notes_multisymbol_backtest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b8f94ee-5f04-430e-adc4-2024f41fa7ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83bef78e-9790-496c-9ba0-41c5a9e7e271",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# uncomment to disable numba\n",
    "# os.environ['NOJIT'] = 'true'\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import pprint\n",
    "from njit_multisymbol import *\n",
    "from procedures import utc_ms, load_live_config, fetch_market_specific_settings, make_get_filepath\n",
    "from pure_funcs import (\n",
    "    date_to_ts2,\n",
    "    ts_to_date_utc,\n",
    "    tuplify,\n",
    "    numpyize,\n",
    "    stats_multi_to_df,\n",
    "    fills_multi_to_df,\n",
    "    calc_drawdowns,\n",
    "    calc_sharpe_ratio,\n",
    "    analyze_fills_multi,\n",
    "    live_config_dict_to_list_recursive_grid,\n",
    ")\n",
    "from njit_funcs import round_dynamic\n",
    "from plotting import plot_fills_multi, plot_pnls_long_short, plot_pnls_separate, plot_pnls_stuck\n",
    "from numba import njit\n",
    "from downloader import load_hlc_cache, prepare_multsymbol_data\n",
    "from backtest_multi import prep_hlcs_mss_config, prep_config_multi, args2config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48c3f7cb-d0d4-4690-b6e1-1cdf708ec836",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = [24, 13.5]\n",
    "plt.rcParams[\"figure.facecolor\"] = \"w\"\n",
    "pd.set_option(\"display.precision\", 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "febb9a6b-3aa5-4910-9e92-6e9ea722492b",
   "metadata": {},
   "source": [
    "### multi symbol backtest with lossless auto unstuck\n",
    "\n",
    "- if a position is stuck, bot will use profits made on other markets to realize losses for the stuck position\n",
    "- if multiple positions are stuck, select the stuck pos whose price action distance is the lowest\n",
    "- each live config's individual auto unstuck is disabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27c24a78-1e60-439d-b150-daa358c1d9dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cfg =  {\n",
    "    \"global\": {\n",
    "        \"TWE_long\": 1.9628217193346074,\n",
    "        \"TWE_short\": 8.873613532492868,\n",
    "        \"loss_allowance_pct\": 0.007057868173311633,\n",
    "        \"stuck_threshold\": 0.8214753608753934,\n",
    "        \"unstuck_close_pct\": 0.0011074274412150183\n",
    "    },\n",
    "    \"long\": {\n",
    "        \"ddown_factor\": 1.3552241067565591,\n",
    "        \"ema_span_0\": 1251.5875524266064,\n",
    "        \"ema_span_1\": 604.1687419843032,\n",
    "        \"enabled\": True,\n",
    "        \"initial_eprice_ema_dist\": -0.020367702184693325,\n",
    "        \"initial_qty_pct\": 0.013372380921445446,\n",
    "        \"markup_range\": 0.003390533528629384,\n",
    "        \"min_markup\": 0.005905020194249954,\n",
    "        \"n_close_orders\": 2.1233372580530903,\n",
    "        \"rentry_pprice_dist\": 0.049046697381191565,\n",
    "        \"rentry_pprice_dist_wallet_exposure_weighting\": 0.5612700282178685,\n",
    "        \"wallet_exposure_limit\": 0.39256434386692146\n",
    "    },\n",
    "    \"short\": {\n",
    "        \"ddown_factor\": 0.3907245872883093,\n",
    "        \"ema_span_0\": 1103.2107410642989,\n",
    "        \"ema_span_1\": 1370.6087776402064,\n",
    "        \"enabled\": False,\n",
    "        \"initial_eprice_ema_dist\": 0.0017511634241226537,\n",
    "        \"initial_qty_pct\": 0.044680299514502905,\n",
    "        \"markup_range\": 0.0017555611262703857,\n",
    "        \"min_markup\": 0.007697306221784713,\n",
    "        \"n_close_orders\": 12.949796081895949,\n",
    "        \"rentry_pprice_dist\": 0.02703134258902625,\n",
    "        \"rentry_pprice_dist_wallet_exposure_weighting\": 2.4981211753897754,\n",
    "        \"wallet_exposure_limit\": 1.7747227064985736\n",
    "    }\n",
    "}\n",
    "starting_balance = 1000000\n",
    "symbols = ['BALUSDT', 'BANDUSDT', 'CELRUSDT', 'OMGUSDT', 'SKLUSDT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b145c92f-de5b-41a8-b7ab-0a6606bea9a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Args:\n",
    "    def __init__(self):\n",
    "        self.symbols = symbols\n",
    "        self.symbols = {s: \"\" for s in self.symbols}\n",
    "        self.user = \"binance_01\"\n",
    "        self.start_date = \"2021-05-01\"\n",
    "        self.end_date = \"now\"\n",
    "        self.starting_balance = starting_balance\n",
    "        self.long_enabled = cfg[\"long\"][\"enabled\"]\n",
    "        self.short_enabled = cfg[\"short\"][\"enabled\"]\n",
    "        self.TWE_long = cfg[\"global\"][\"TWE_long\"]\n",
    "        self.TWE_short = cfg[\"global\"][\"TWE_short\"]\n",
    "\n",
    "\n",
    "args = Args()\n",
    "config = args2config(args)\n",
    "config[\"base_dir\"] = \"backtests\"\n",
    "config[\"symbols\"] = tuple(sorted(set(config[\"symbols\"])))\n",
    "hlcs, mss, config = await prep_hlcs_mss_config(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64df6915-9365-4b18-800d-8d9dfd866df9",
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols = tuple(sorted(set(config[\"symbols\"])))\n",
    "\n",
    "# specify live configs for each symbol. Use a single live config, or load separately for each symbol.\n",
    "# live_configs = {symbol: load_live_config(f\"configs/live/multisymbol/no_AU/{symbol}.json\") for symbol in symbols}\n",
    "live_configs = {symbol: cfg for symbol in symbols}\n",
    "for s in live_configs:\n",
    "    live_configs[s][\"long\"][\"wallet_exposure_limit\"] = cfg[\"global\"][\"TWE_long\"] / len(symbols)\n",
    "    live_configs[s][\"short\"][\"wallet_exposure_limit\"] = cfg[\"global\"][\"TWE_short\"] / len(symbols)\n",
    "live_configs_np = numpyize(\n",
    "    [live_config_dict_to_list_recursive_grid(live_configs[s]) for s in symbols]\n",
    ")\n",
    "\n",
    "do_longs = tuplify([cfg[\"long\"][\"enabled\"] for s in config[\"symbols\"]])\n",
    "do_shorts = tuplify([cfg[\"short\"][\"enabled\"] for s in config[\"symbols\"]])\n",
    "qty_steps = tuplify([mss[symbol][\"qty_step\"] for symbol in config[\"symbols\"]])\n",
    "price_steps = tuplify([mss[symbol][\"price_step\"] for symbol in config[\"symbols\"]])\n",
    "min_costs = tuplify([mss[symbol][\"min_cost\"] for symbol in config[\"symbols\"]])\n",
    "min_qtys = tuplify([mss[symbol][\"min_qty\"] for symbol in config[\"symbols\"]])\n",
    "c_mults = tuplify([mss[symbol][\"c_mult\"] for symbol in config[\"symbols\"]])\n",
    "maker_fee = next(iter(mss.values()))[\"maker\"]\n",
    "starting_balance = config[\"starting_balance\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c33fb11-4bbb-4b8b-830a-a0be61a971bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "hlcs_clipped = hlcs  # [:,0:60000] # to backtest on subset on data\n",
    "# will compile JIT on the first run, then be faster on subsequent runs\n",
    "sts = utc_ms()\n",
    "res = backtest_multisymbol_recursive_grid(\n",
    "    hlcs_clipped,\n",
    "    starting_balance,\n",
    "    maker_fee,\n",
    "    do_longs,\n",
    "    do_shorts,\n",
    "    c_mults,\n",
    "    symbols,\n",
    "    qty_steps,\n",
    "    price_steps,\n",
    "    min_costs,\n",
    "    min_qtys,\n",
    "    live_configs_np,\n",
    "    cfg[\"global\"][\"loss_allowance_pct\"],\n",
    "    cfg[\"global\"][\"stuck_threshold\"],\n",
    "    cfg[\"global\"][\"unstuck_close_pct\"],\n",
    ")\n",
    "print(f\"time elapsed for backtest {(utc_ms() - sts) / 1000:.6f}s\")\n",
    "fills, stats = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0e11000-61c4-4bdb-8a90-6f14f11bca25",
   "metadata": {},
   "outputs": [],
   "source": [
    "sts = utc_ms()\n",
    "fdf = fills_multi_to_df(fills, symbols, c_mults)\n",
    "sdf = stats_multi_to_df(stats, symbols, c_mults)\n",
    "print(f\"time elapsed for analysis {(utc_ms() - sts) / 1000:.6f}s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c04f913b-1a34-4c4b-84fe-2e3ebe93493a",
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\"TWE_long\": cfg[\"global\"][\"TWE_long\"], \"TWE_short\": cfg[\"global\"][\"TWE_short\"]}\n",
    "params = cfg[\"global\"]\n",
    "analysis = analyze_fills_multi(sdf, fdf, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c0f4cd-930d-412e-896e-f3ec3111c1b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "mkl = max([len(k) for k in analysis])\n",
    "for k, v in analysis.items():\n",
    "    if isinstance(v, dict):\n",
    "        continue\n",
    "        mkls = max([len(s) for s in v])\n",
    "        for symbol in v:\n",
    "            mkl1 = max([len(k) for k in v[symbol]])\n",
    "            for k1, v1 in v[symbol].items():\n",
    "                print(f\"    {symbol: <{mkls}} {k1: <{mkl1}} {round_dynamic(v1, 6)}\")\n",
    "            print()\n",
    "    else:\n",
    "        print(f\"{k: <{mkl}} {round_dynamic(v, 6)}\")\n",
    "adf = pd.DataFrame({k: v for k, v in analysis[\"individual_analyses\"].items()})\n",
    "adf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d19f2af8-8c79-4066-b27d-43667f174a61",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not adf.T.upnl_pct_min_long.isna().all():\n",
    "    print(\"upnl pct min long\")\n",
    "    print(adf.T.upnl_pct_min_long.sort_values())\n",
    "    print()\n",
    "if not adf.T.upnl_pct_min_short.isna().all():\n",
    "    print(\"upnl pct min short\")\n",
    "    print(adf.T.upnl_pct_min_short.sort_values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08b8fd86-00bf-49ad-957f-3a03e8cfb101",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not (adf.T.loss_profit_ratio_long == 1.0).all():\n",
    "    print(\"loss_profit_ratio_long\")\n",
    "    print(adf.T.loss_profit_ratio_long.sort_values(ascending=False))\n",
    "    print()\n",
    "if not (adf.T.loss_profit_ratio_short == 1.0).all():\n",
    "    print(\"loss_profit_ratio_short\")\n",
    "    print(adf.T.loss_profit_ratio_short.sort_values(ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e15b8b4-4f23-4f83-ad42-c5f609d39c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "adf.T.pnl_ratio.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6271c145-b321-4dd2-90a1-a182af9c106b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find worst performers\n",
    "worsts = []\n",
    "clip = 0.4\n",
    "for x in [\n",
    "    adf.T.pnl_ratio.sort_values(),\n",
    "    # adf.T.loss_profit_ratio_short.sort_values(ascending=False),\n",
    "    adf.T.loss_profit_ratio_long.sort_values(ascending=False),\n",
    "    # adf.T.upnl_pct_min_short.sort_values(),\n",
    "    # adf.T.upnl_pct_min_long.sort_values(),\n",
    "]:\n",
    "    worsts.append(list(dict(x.iloc[: int(len(x) * clip)])))\n",
    "to_drop = [x for x in worsts[0] if all([x in w for w in worsts])]\n",
    "to_drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b5c7218-482c-4d83-aeb9-201233caa15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1539d10f-3e9c-47e9-b770-2094615222bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88ff94bb-c813-44af-9646-e5171df994fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot drawdowns\n",
    "min_multiplier = 60 * 24\n",
    "drawdowns = calc_drawdowns(sdf.equity)\n",
    "drawdowns_daily = drawdowns.groupby(drawdowns.index // min_multiplier * min_multiplier).min()\n",
    "drawdowns_ten_worst = drawdowns_daily.sort_values().iloc[:10]\n",
    "print(drawdowns_ten_worst)\n",
    "drawdowns_ten_worst.plot(style=\"ro\")\n",
    "drawdowns.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e3eb28f-cfb9-4be3-9596-dd77980e3dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_pnls_stuck(sdf, fdf, start_pct=0.0, end_pct=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfb98601-253b-4bb8-86cf-87ebffa6863a",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_pnls_separate(sdf, fdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98fb948d-dff6-47c5-b733-bc2b46d9d18c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_pnls_long_short(sdf, fdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71507db2-3198-449f-aaac-75f70b2b8b12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inspect two months before and two months after location of worst drawdown\n",
    "drawdowns = calc_drawdowns(sdf.equity)\n",
    "worst_drawdown_loc = drawdowns.sort_values().iloc[:1].index[0]\n",
    "wdls = worst_drawdown_loc - 60 * 24 * 30 * 2\n",
    "wdle = worst_drawdown_loc + 60 * 24 * 30 * 2\n",
    "sdfc = sdf.loc[wdls:wdle]\n",
    "sdfc.balance.plot()\n",
    "sdfc.equity.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c10cdc8-8963-4343-8321-67afc4a35b71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inspect for each symbol\n",
    "for symbol in symbols:\n",
    "    print(symbol)\n",
    "    plot_fills_multi(symbol, sdf.loc[wdls:wdle], fdf.loc[wdls:wdle]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db32cfb6-9e36-430f-ab9b-86720bfb31ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inspect individual lowest drawdowns\n",
    "upnl_pct_mins = sdf[[c for c in sdf.columns if \"upnl\" in c]].min().sort_values()\n",
    "print(upnl_pct_mins)\n",
    "print()\n",
    "n = 60 * 24 * 60\n",
    "upnl_pct_idxs = dict(sdf[[c for c in sdf.columns if \"upnl\" in c]].idxmin())\n",
    "for sym in dict(upnl_pct_mins):\n",
    "    idx = upnl_pct_idxs[sym]\n",
    "    if np.isnan(idx):\n",
    "        continue\n",
    "    print(sym, idx)\n",
    "    plot_fills_multi(\n",
    "        sym[: sym.find(\"_\")], sdf.loc[idx - n : idx + n], fdf.loc[idx - n : idx + n]\n",
    "    ).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "249dbbc9-bb6c-4ab0-8f67-98b92c106be9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# exposures\n",
    "sdf[[c for c in sdf.columns if \"WE\" in c]].sum(axis=1).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d06c6963-f3c2-49de-b9e0-53b63c65a05f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find worst realized losses for each symbol\n",
    "wdds = {}\n",
    "mins_before_and_after = 60 * 24 * 30 * 1\n",
    "for symbol in symbols:\n",
    "    wdd = calc_drawdowns(fdf[fdf.symbol == symbol].pnl.cumsum() + sdf.balance.iloc[0])\n",
    "    wdds[symbol] = [abs(wdd.min()), wdd.idxmin()]\n",
    "for symbol, wdd in sorted(wdds.items(), key=lambda x: x[1][0], reverse=True):\n",
    "    print(symbol, f\"pct loss: {wdd[0] * 100:.2f}% n_days: {wdd[1] / (60 * 24):.2f}\")\n",
    "    fdfc = fdf.loc[wdd[1] - mins_before_and_after : wdd[1] + mins_before_and_after]\n",
    "    sdfc = sdf.loc[wdd[1] - mins_before_and_after : wdd[1] + mins_before_and_after]\n",
    "    plot_fills_multi(symbol, sdfc, fdfc).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "775d12cc-fb1d-400c-bbb9-3131df23b0e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}