examples/benchmarks/TRA/Reports.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview\n",
    "\n",
    "This notebook contains all experiment results exhibited in our paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "sns.set(style='white')\n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "def func(x, N=80):\n",
    "    ret = x.ret.copy()\n",
    "    x = x.rank(pct=True)\n",
    "    x['ret'] = ret\n",
    "    diff = x.score.sub(x.label)\n",
    "    r = x.nlargest(N, columns='score').ret.mean()\n",
    "    r -= x.nsmallest(N, columns='score').ret.mean()\n",
    "    return pd.Series({\n",
    "        'MSE': diff.pow(2).mean(), \n",
    "        'MAE': diff.abs().mean(), \n",
    "        'IC': x.score.corr(x.label),\n",
    "        'R': r\n",
    "    })\n",
    "    \n",
    "ret = pd.read_pickle(\"data/ret.pkl\").clip(-0.1, 0.1)\n",
    "def backtest(fname, **kwargs):\n",
    "    pred = pd.read_pickle(fname).loc['2018-09-21':'2020-06-30']  # test period\n",
    "    pred['ret'] = ret\n",
    "    dates = pred.index.unique(level=0)\n",
    "    res = Parallel(n_jobs=-1)(delayed(func)(pred.loc[d], **kwargs) for d in dates)\n",
    "    res = {\n",
    "       dates[i]: res[i]\n",
    "       for i in range(len(dates))\n",
    "    }\n",
    "    res = pd.DataFrame(res).T\n",
    "    r = res['R'].copy()\n",
    "    r.index = pd.to_datetime(r.index)\n",
    "    r = r.reindex(pd.date_range(r.index[0], r.index[-1])).fillna(0)  # paper use 365 days\n",
    "    return {\n",
    "        'MSE': res['MSE'].mean(),\n",
    "        'MAE': res['MAE'].mean(),\n",
    "        'IC': res['IC'].mean(),\n",
    "        'ICIR': res['IC'].mean()/res['IC'].std(),\n",
    "        'AR': r.mean()*365,\n",
    "        'AV': r.std()*365**0.5,\n",
    "        'SR': r.mean()/r.std()*365**0.5,\n",
    "        'MDD': (r.cumsum().cummax() - r.cumsum()).max()\n",
    "    }, r\n",
    "\n",
    "def fmt(x, p=3, scale=1, std=False):\n",
    "    _fmt = '{:.%df}'%p\n",
    "    string = _fmt.format((x.mean() if not isinstance(x, (float, np.floating)) else x) * scale)\n",
    "    if std and len(x) > 1:\n",
    "        string += ' ('+_fmt.format(x.std()*scale)+')'\n",
    "    return string\n",
    "\n",
    "def backtest_multi(files, **kwargs):\n",
    "    res = []\n",
    "    pnl = []\n",
    "    for fname in files:\n",
    "        metric, r = backtest(fname, **kwargs)\n",
    "        res.append(metric)\n",
    "        pnl.append(r)\n",
    "    res = pd.DataFrame(res)\n",
    "    pnl = pd.concat(pnl, axis=1)\n",
    "    return {\n",
    "        'MSE': fmt(res['MSE'], std=True),\n",
    "        'MAE': fmt(res['MAE'], std=True),\n",
    "        'IC': fmt(res['IC']),\n",
    "        'ICIR': fmt(res['ICIR']),\n",
    "        'AR': fmt(res['AR'], scale=100, p=1)+'%',\n",
    "        'VR': fmt(res['AV'], scale=100, p=1)+'%',\n",
    "        'SR': fmt(res['SR']),\n",
    "        'MDD': fmt(res['MDD'], scale=100, p=1)+'%'\n",
    "    }, pnl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation\n",
    "\n",
    "\n",
    "You could prepare the source data as below for the backtest code:\n",
    "1. Linear: see Qlib examples\n",
    "2. LightGBM: see Qlib examples\n",
    "3. MLP: see Qlib examples\n",
    "4. SFM: see Qlib examples\n",
    "5. ALSTM: `qrun` configs/config_alstm.yaml\n",
    "6. Transformer: `qrun` configs/config_transformer.yaml\n",
    "7. ALSTM+TRA: `qrun` configs/config_alstm_tra_init.yaml && `qrun` configs/config_alstm_tra.yaml\n",
    "8. Tranformer+TRA: `qrun` configs/config_transformer_tra_init.yaml && `qrun` configs/config_transformer_tra.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = {\n",
    "    'Linear': ['output/Linear/pred.pkl'],\n",
    "    'LightGBM': ['output/GBDT/lr0.05_leaves128/pred.pkl'],\n",
    "    'MLP': glob.glob('output/search/MLP/hs128_bs512_do0.3_lr0.001_seed*/pred.pkl'),\n",
    "    'SFM': glob.glob('output/search/SFM/hs32_bs512_do0.5_lr0.001_seed*/pred.pkl'),\n",
    "    'ALSTM': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
    "    'Trans.': glob.glob('output/search/Transformer/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
    "    'ALSTM+TS':glob.glob('output/LSTM_Attn_TS/hs256_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
    "    'Trans.+TS':glob.glob('output/Transformer_TS/head4_hs64_bs1024_do0.1_lr0.0002_seed*/pred.pkl'),\n",
    "    'ALSTM+TRA(Ours)': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
    "    'Trans.+TRA(Ours)': glob.glob('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb1.0_head4_hs64_bs512_do0.1_lr0.0005_seed*/pred.pkl')\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0acd535e05944e539fd001009ed0748d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {\n",
    "    name: backtest_multi(exps[name])\n",
    "    for name in tqdm(exps)\n",
    "}\n",
    "report = pd.DataFrame({\n",
    "    k: v[0]\n",
    "    for k, v in res.items()\n",
    "}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>IC</th>\n",
       "      <th>ICIR</th>\n",
       "      <th>AR</th>\n",
       "      <th>VR</th>\n",
       "      <th>SR</th>\n",
       "      <th>MDD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Linear</th>\n",
       "      <td>0.163</td>\n",
       "      <td>0.327</td>\n",
       "      <td>0.020</td>\n",
       "      <td>0.132</td>\n",
       "      <td>-3.2%</td>\n",
       "      <td>16.8%</td>\n",
       "      <td>-0.191</td>\n",
       "      <td>32.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LightGBM</th>\n",
       "      <td>0.160</td>\n",
       "      <td>0.323</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.292</td>\n",
       "      <td>7.8%</td>\n",
       "      <td>15.5%</td>\n",
       "      <td>0.503</td>\n",
       "      <td>25.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MLP</th>\n",
       "      <td>0.160 (0.002)</td>\n",
       "      <td>0.323 (0.003)</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.273</td>\n",
       "      <td>3.7%</td>\n",
       "      <td>15.3%</td>\n",
       "      <td>0.264</td>\n",
       "      <td>26.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SFM</th>\n",
       "      <td>0.159 (0.001)</td>\n",
       "      <td>0.321 (0.001)</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.381</td>\n",
       "      <td>7.1%</td>\n",
       "      <td>14.3%</td>\n",
       "      <td>0.497</td>\n",
       "      <td>22.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALSTM</th>\n",
       "      <td>0.158 (0.001)</td>\n",
       "      <td>0.320 (0.001)</td>\n",
       "      <td>0.053</td>\n",
       "      <td>0.419</td>\n",
       "      <td>12.3%</td>\n",
       "      <td>13.7%</td>\n",
       "      <td>0.897</td>\n",
       "      <td>20.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trans.</th>\n",
       "      <td>0.158 (0.001)</td>\n",
       "      <td>0.322 (0.001)</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.400</td>\n",
       "      <td>14.5%</td>\n",
       "      <td>14.2%</td>\n",
       "      <td>1.028</td>\n",
       "      <td>22.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALSTM+TS</th>\n",
       "      <td>0.160 (0.002)</td>\n",
       "      <td>0.321 (0.002)</td>\n",
       "      <td>0.039</td>\n",
       "      <td>0.291</td>\n",
       "      <td>6.7%</td>\n",
       "      <td>14.6%</td>\n",
       "      <td>0.480</td>\n",
       "      <td>22.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trans.+TS</th>\n",
       "      <td>0.160 (0.004)</td>\n",
       "      <td>0.324 (0.005)</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.278</td>\n",
       "      <td>10.4%</td>\n",
       "      <td>14.7%</td>\n",
       "      <td>0.722</td>\n",
       "      <td>23.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALSTM+TRA(Ours)</th>\n",
       "      <td>0.157 (0.000)</td>\n",
       "      <td>0.318 (0.000)</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.460</td>\n",
       "      <td>12.4%</td>\n",
       "      <td>14.0%</td>\n",
       "      <td>0.885</td>\n",
       "      <td>20.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trans.+TRA(Ours)</th>\n",
       "      <td>0.157 (0.000)</td>\n",
       "      <td>0.320 (0.000)</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.442</td>\n",
       "      <td>16.1%</td>\n",
       "      <td>14.2%</td>\n",
       "      <td>1.133</td>\n",
       "      <td>23.1%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            MSE            MAE     IC   ICIR     AR     VR  \\\n",
       "Linear                    0.163          0.327  0.020  0.132  -3.2%  16.8%   \n",
       "LightGBM                  0.160          0.323  0.041  0.292   7.8%  15.5%   \n",
       "MLP               0.160 (0.002)  0.323 (0.003)  0.037  0.273   3.7%  15.3%   \n",
       "SFM               0.159 (0.001)  0.321 (0.001)  0.047  0.381   7.1%  14.3%   \n",
       "ALSTM             0.158 (0.001)  0.320 (0.001)  0.053  0.419  12.3%  13.7%   \n",
       "Trans.            0.158 (0.001)  0.322 (0.001)  0.051  0.400  14.5%  14.2%   \n",
       "ALSTM+TS          0.160 (0.002)  0.321 (0.002)  0.039  0.291   6.7%  14.6%   \n",
       "Trans.+TS         0.160 (0.004)  0.324 (0.005)  0.037  0.278  10.4%  14.7%   \n",
       "ALSTM+TRA(Ours)   0.157 (0.000)  0.318 (0.000)  0.059  0.460  12.4%  14.0%   \n",
       "Trans.+TRA(Ours)  0.157 (0.000)  0.320 (0.000)  0.056  0.442  16.1%  14.2%   \n",
       "\n",
       "                      SR    MDD  \n",
       "Linear            -0.191  32.1%  \n",
       "LightGBM           0.503  25.7%  \n",
       "MLP                0.264  26.2%  \n",
       "SFM                0.497  22.9%  \n",
       "ALSTM              0.897  20.2%  \n",
       "Trans.             1.028  22.5%  \n",
       "ALSTM+TS           0.480  22.3%  \n",
       "Trans.+TS          0.722  23.7%  \n",
       "ALSTM+TRA(Ours)    0.885  20.4%  \n",
       "Trans.+TRA(Ours)   1.133  23.1%  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report\n",
    "# print(report.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ1\n",
    "\n",
    "Case study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAADMCAYAAACx8ZDiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABZ7ElEQVR4nO2dd3hUVfrHP9PSE9IbNSCEriAgKh2BnwqCriAiiCsqdlBXZXGXoqiwFkRdV0WsgCICIkEpIlXpHRICoQfSezKTTGbm/P4YMiQkgUyYmjmf5+Fhcst7vvPeufe9p71HIYQQSCQSiUQicWmUzhYgkUgkEonk2siALZFIJBKJGyADtkQikUgkboAM2BKJRCKRuAEyYEskEolE4gaonS3A1pSWlnLkyBEiIiJQqVTOliOR2Ayj0UhWVhYdO3bEx8fH2XKcgry/JQ2Za93jDS5gHzlyhIceesjZMiQSu7Fo0SK6devmbBlOQd7fEk+gtnvc4QH79OnTTJkyhfz8fIKDg5kzZw4tWrSocsyyZcv4+uuvUSqVmEwmRo4cycMPP1wn+xEREYD5C0dHR9tavkTiNNLT03nooYcsv3FPRN7fkobMte5xhwfs6dOnM2bMGIYPH87KlSuZNm0a3377bZVjhgwZwn333YdCoaC4uJhhw4bRo0cP2rZte037Fc1k0dHRNGnSxC7fQSJxJp7cFCzvb4knUNs97tBBZzk5OSQmJjJ06FAAhg4dSmJiIrm5uVWOCwgIQKFQAOY+q/LycsvfEonE/ViyZAnDhw9nwIABzJs3z9lyJBK3xKEBOy0tjaioKMvbg0qlIjIykrS0tGrHbtiwgbvvvpv+/fvz2GOPER8f70ipEonERqxZs4adO3fy008/kZCQwNKlS8nMzHS2LIkjyMmBvn1h/37z/2fOOFuRW+Oyg84GDhzIwIEDuXjxIs888wx9+vShZcuW9bZXXl5OamoqpaWlNlTpPHx8fGjSpAkajcbZUpzKxaIMtp7ZxaiOQ2UrjAtiMpmYO3cuS5YsQaPRoNFoiIqK4tSpU0RGRjpbnsTefP01bNsGo0dDSgpMnAhr1zpbldvi0IAdExNDRkYGRqMRlUqF0WgkMzOTmJiYWs+JjY2lU6dObNq06boCdmpqKoGBgbRo0cLtH+xCCHJyckhNTSUuLs7ZcpzK2hOb+e3ERvrH3UpkQLiz5UiuYN++fWRnZzN+/HjLtpSUFEJDQ52oSuIQhIC5c8FkguPHzdu2bYPffoM773SuNjfFoU3iYWFhtGvXjoSEBAASEhJo165dtZv35MmTls+5ubns3LmTNm3aXFfZpaWlhIWFuX2wBlAoFISFhTWY1oLrISnrBABpxbKJ1RU5cuQIo0aNYuXKlaxcuZJ3330XHx8fTCYT06ZN47XXXmPKlCnIRQMbIFu3QkFB1W1aLUyYAPLZVS8cnulsxowZLFy4kCFDhrBw4UJmzpwJwOOPP87hw4cB8wCVu+++m+HDh/PII48wduxYevXqdd1lN4RgXUFD+i71RavXcTb/AgBpRTJguyJ5eXn4+vpa/l6zZg0DBw6kbdu2vP7667z55ptotVq0Wq0TVUrswgcfQElJ9e0FBTB7tsPlNAQc3ofdqlUrli5dWm37/PnzLZ+nTp3qSEkSN+VY9kkE5ppZugzYLklcXBwrVqwA4MSJEyxbtozvv/8egB07drB06VJCQkKqBHVJAyE52dwsfiVaLaxeDTNmOFySuyNziUvclmPZKagUSmICI0krznK2HEkNDBkyBLVazYABA5g6dSpz5861jFnp2bMn7733Hmq1mqSkJCcrldico0fNAbumf7t3O1udW+Kyo8QlnofJZMKEQK2sW2KQpKwUWoY2J8wvhDN55+2sTlIffH19WbBgQbXtO3fuZO3atQghMBgMtG7d2gnqJBL3QgZsJ7NkyRIWL15MUVERw4cPZ9KkSc6W5DS+PrCUbWd383zPR7kppv1Vj9Ub9KTknuHuNgNQKVTsSj2AwWSsc7CXOJdbbrmFW265xdkyJBK3QgZsJ1I5oUR5eTmDBw/mwQcf9Mj5qSV6LX+c+hOjMPH2lo8Z1nYQbcLiCPT2J8Q3mFDfYLxUl+ecp+SewWgy0i6iNUVlxZiEiaySHGIC3dN35XnpaE/swS++B5pG7vkdJBKJfZEB20nIhBJV2XxmB3pjOa8PeIl1J7fyy7F11Y4J8PInzDeYUL9gistKUKAgPrwlFwrTAfNIcXcL2EZtERnL36H07FFAgTo4SgZsiURSIx4bsP/Yc471u87ZxfagHs0Y0K3ZVY+RCSUuI4Rg/cmt3BDagrYRN9A24gbG3ngvhaVFFJQVkacrIFeXT442j1xd/qV/BXSN7UiAlz8xAeYAl+6Gc7FzNy2i9FwSIf0eIrBTX9RBYc6WJJFIXBSPDdjOpiKhxKuvvgqYp7yMHj2amJgY/vnPf/LXX3+xefNmJ6t0DElZKVwoTOep7uMs20IvNYPXhUDvAPw0vlwsyrCTQvtQdjGFov2/06jH3YTcfp+z5UgkEhfHYwP2gG7XrgXbk9oSSgQGBvL222/zyCOPOE2bIxBCsPjQz6QWppFamI6/xpfbmlVfsL0uKBQKYgIiSS+qOrWrRK/FS6VBo3K9fOtCmMhe+wUq/0aE9HnA2XIkEokb4LEB29lcLaGEJ3A08zgrj60jJjCSIC9/hsXfgbfaq972YgIjSc45BUBWSQ4rj61j46m/aB/Zmql9nnO5zHDaE3spu3iCiGHPofT2c7YciUTiBsiA7SSGDBnCqlWrGDBgAGFhYVUSSngCCcc3EOQdwDtD/lVl9Hd9iQ6M5M9ze/jl2DqWHF6FCUHb8FYcTE9iw6k/uaPV9ae2tSXleeaBcn5tujtZiUQicRdkwHYStSWUAJg5cyanTp1i2rRpPP744zRt2tTB6uzLhcJ09l08zMgOd9skWAPEBEQiECw8uIJusZ159OYHCPUN5o1N8/juwDJuim5PuL/rDOgzaQtBoZS1a4lEUmdkalIXZPr06WzZsoXXX3+9wQVrgNXH/0CjVDP4hj42s9ku8gbiQpoysdtDvNzrScL9QlEqlDzVfRwmBJ/tWVSnFaF+P7mNn5Psv16vUVeMyi/Q5ZrqJRKJ6yIDtsShFJYWsfnMDvq06EkjnyCb2Q33C2XO4KkMbNWrShCMDAjnoc4jOJieyKbT269pZ/3JLaw9Yf/R+SZdIUrfQLuXI5FIGg6ySVziUNad3EK5sZy74wc4rMzBN/Rh+/l9fHPgJ26Mbk+oX3CNx5mEiYuFGZQZ9ZQZ9Nc1CO5aGLVFqPxs98JiL/Lz8/nyyy9JSkqqtgTmokWLnKRKIvFMZMCWOAy9sZy1JzbTJaYjTYIcN8DO3DQ+ln+sncXnexbxau+na2yKztXmU2bUA5BRnEWz4MZ202TUFaIJjbWbfVvx0ksvodfrufPOO+USmBKJk5EBW+Iwtp3dRUFZEUPjBzq87OjASB7sNJxvDvzE1rO76NOi+sITqZdSnAKkFWfaNWCbtEWomrh+DXv//v3s2LEDLy/7tTZIJJK6IfuwJQ5BCEFC8gaaBzehY2S8UzTc2bo/8WEt+Wr/j+TpCqrtv1hUKWAX2S/NqRACo7YQlRv0YcfHx5Oenn7tAyUSid2RNWyJQziYnkhqYRrP9BjvtJHRSqWSp3qM4+V1bzF/7/e8fPvEKlpSC9Px9/JDrVCRXpx1FUvXh6lMC8KE0g36sHv27Mljjz3GfffdR3h4eJV9999/v5NUSSSeiQzYEoeQkLyBEJ9G3F7P9KO2IjYomgc6DmPhweWsS9nCkNZ9LfsuFKbTJDAaFArS7VjDNumKANyihr1nzx6ioqL4888/q2xXKBQyYEskDua6AnZpaSlKpVL2b0muytn8VA5lJPFgp+GoVc5/RxwaP5DErBN8vf9HmjSKoUNkGwAuFqZzc2wnTEJwMCPRbuUbtYUAqPxcP2B/9913zpYgkUguYVUf9pw5czh06BAAmzZtokePHnTv3p0//vjDLuIkDYPVyX/grfJiUKvezpYCmEeNP3/L34kOiOT9Pz8nqySH4rISCsqKaBwUQ3RgBHm6AkoNZXYp36Q117CVvq7fJA5QUFDAzz//zGeffcbPP/9MQUH1/n+JRGJ/rArYq1atonXr1gD897//5Z133uF///sfc+fOtYs4T2DJkiUMHz6cAQMGMG/ePGfLsTl5ugK2nttFv7hbCfD2d7YcC35evrzS+yn0xnKWHF5lGSHeOCiK6Evra2fYqR/bqHOfGvb+/fsZNGgQP/zwA8nJyfzwww8MGjSI/fv3O1uaROJxWNU+qdPp8PX1JS8vj/PnzzNkyBAALly4YBdxDZ01a9awc+dOfvrpJ8rLyxk8eDAPPvggkZGRzpZmMzae/guTycTdbRyXKKWuxARGMqhVb349sdGSTKVxUDTa8lLAPFK8eXATm5drvFTDdofEKW+99RbTp0/n7rvvtmz79ddfmTVrFsuWLXOiMonE87Cqht2iRQt++eUXFi1axO233w5Abm4uPj4+dhHXkDGZTMydO5dp06ah0Wjw8/MjKiqKU6dOOVuaTcnR5hHo7U90oGu+hAxtewcqhZJVx9ajUWmI8AsjOiACwG4jxU26QlCqUXi5fiKSM2fOcOedd1bZNmTIEM6dO+ckRRKJ52JVDXv69Om89dZbaDQa3nzzTQC2bdtmCd7uRNGhTRQdtE/fe+CNAwjs3O+qx+zbt4/s7GzGjx9v2ZaSkkJoqOusKGULjCYjaqXzB5rVRqhvMP1b3sa6lC00CYxFqVTiq/ShkU+Q3eZiG7VFqHwD3GLhj+bNm7N69WqGDRtm2bZmzZoGuSiNROLqWPUk7dy5Mz/88EOVbffccw/33HOPTUV5AkeOHGHUqFG8+uqrAJw4cYLRo0dTUFDA1KlT0ev1BAUFMW3aNCcrvT4MJiMqpcrZMq7K8LaD2XByG42Doi3bYgIiSC+2U8DWFbnFHGyAqVOn8uSTT/Ldd98RGxvLhQsXOHv2LJ9++qmzpUkkHodVAXvHjh00btyYpk2bkpmZyXvvvYdSqeTFF18kIiLCXhrtQmDnftesBduTvLy8KrmZ16xZw8CBA+nevTvdu3cH4KmnnqKkpAR/f9cZrGUtBpMBtYsH7Aj/MF7p/RRR/pcTg0QHRrL/4hGMdnjhMGkL3WLAGUDXrl1Zv349mzZtIjMzk/79+9O3b1+Cg4OdLU0i8Tis6sOeOXMmKpX54TVnzhwMBgMKhYJ///vfdhHXkImLi2Pv3r2AuXa9bNkyXnjhBcv+TZs20apVK7cO1gAG4dpN4hV0ielIbKUadsfIeArKinhry0cUlhXbtCyjrgiVm0zpAmjUqBHDhw/n8ccfZ/jw4TJYSyROwqonaUZGBrGxsRgMBrZt28Yff/yBRqOhd2/XmF/rTgwZMoRVq1YxYMAAwsLCmDt3LjEx5hWsli9fzoULF/jHP/7hZJXXj8FkRK1w7Rp2TfRpcQsmYWL+nsX8c93bvHT7RFqGNrOJbaO2EKUL17AnTJjAggULABgzZkytfe1yeU2JxLFYFbADAgLIzs7mxIkTltqfXq/HYDDYS1+DxdfX1/JQrMzGjRv54IMP6NevH9OmTWPy5MluPRDNPOjM/QI2QL+4W2naKJZ3t33Gv/94l4ndHqpxlS9rEMKESVdsqWHnFOj4fl0yfbs0odMN4dc42zGMGDHC8nnkyJHOEyKRSKpgVcAeO3Ys999/P+Xl5UydOhUwj3Zu2bKlXcR5Iv3796d///7OlmEzDCaDyw86uxqtQpsze/AU5v71BR/v/JqTuWcZd9Pf6v0SYiotAWFC5RdIek4J//r0LzJytazdcZYB3ZryxIhO+PtqbPwtrKPyiPCWLVty4403VjumIuOhRCJxHFYF7CeeeIJBgwahUqlo1szcPBgVFcWsWbPsIk7i/rj6tK660MgniH/1m8TCg8v59fgfnMlP5damXblYmEG7yBu4tenNdbZVkTSl2OTN1I+3Um4w8dbTt7M/OZPlG1O4sXU4A7rZpundFvz9739n37591bY/9thj7Nq1ywmKJBLPxeonadOmTdm/fz+HDh0iKiqKLl26oFa79wNZYj8MJiM+am9ny7hu1EoVj3QZSauQ5ny2ZyFJWScAOJqZbFXANl1KS3osvZzcwnI+eKEvrZoE06lVOPcPaI23xjVaI0wmE0KIKv8qOHfunGXwqUQicRxWRdqTJ0/y1FNPUVpaSkxMDGlpaXh7e/Ppp5/SqlUre2mUuDHu3iR+Jb1b9ODGmPYYTUZWHlvHhpPbEELUOQlKRQ07U6vE11tNy8aNLPv8fJzbFF6Z9u3bW75T+/btq+xTKpU8+eSTzpAlkXg0VgXsmTNnMmrUKCZMmGC5mRcsWMCMGTPqvAzf6dOnmTJlCvn5+QQHBzNnzhxatGhR5Zj//ve//Prrr6hUKtRqNS+88IIcie6mNIQm8SsJ8g4AINI/jDKjnsKyIhr51G2aVsVa2BeKFMSE+btstrMNGzYghGDcuHEsXLjQsl2hUBAaGirTEUskTsCqJ+mxY8f46quvqjxkxo8fb1XWo+nTpzNmzBiGDx/OypUrmTZtGt9++22VYzp37syjjz6Kr68vx44dY+zYsWzbtk0+JNwQd8h0Vl8iLyVaySzJqXPArlgL+2yuIDbWz27arpfGjRsD5oQ+SqUSjeZy7b+8vBy9Xo+Xl5ez5EkkHolViVMiIyOrDTTZs2dPnVeXysnJITExkaFDhwIwdOhQEhMTyc3NrXJc7969LVnA4uPjEUKQn59vjVSJi2BOnNJQA3YYAJkl2XU+x6grApWG1Dw90aGunxTn0Ucf5ejRo1W2HT16lAkTJjhJkUTiuVhVw37hhRd4+umn6devH7GxsVy8eJFNmzbxzjvv1On8tLQ0oqKiLANWVCoVkZGRpKWl1TrX+Oeff6ZZs2ZER0fXuF/i2phTkzasJvEKLAG7OKfO55i0hSh8AjAYBdHhrh+wk5OTq03r6ty5M8eOHXOSIonEc7Gqhj1w4ECWL19O69atKSkpoXXr1ixfvpw77rjDLuJ27drFvHnzeO+99+xiX2J/3DXTWV3w0fgQ5B1AZkndA7ZRV4RBbQ7UMWGu2yReQVBQENnZVVsQsrOzq+TBl0gkjsHqqk9cXBxPP/205W+9Xk+/fv3YtGnTNc+NiYkhIyMDo9GISqXCaDSSmZlpSclZmf379/Pyyy/zySefyMQsbow7ZzqrC5H+4dY1iWuL0CvNYzGiw1y/hj148GBeeukl/vWvf9G0aVPOnTvH7Nmzq62RLZFI7I9VNezaSE9Pr9NxYWFhtGvXjoSEBAASEhJo165dtebwQ4cO8cILL/Dhhx/SoUMHW0iUOImGNq3rSiL9w6yqYZt0RRQLb1RKBRHBrl9LfeGFF2jVqhUjR46kS5cuPPDAA8TFxfHiiy86W5pE4nHYpHPRmqkpM2bMYMqUKXzyyScEBQUxZ84cAB5//HGef/55OnXqxMyZMyktLa2yFvR//vMf4uPjbSHXpViyZAmLFy+mqKiI4cOHM2nSJGdLsikNcVpXZSIDwtl54QAmkwml8trvv0ZdEYXKSCJD/FCpbPK+bFe8vb2ZPn0606ZNIy8vj5CQEJediiaRNHQc/iRt1aoVS5curbZ9/vz5ls/Lli1zpCSnsWbNGnbu3MlPP/1EeXk5gwcP5sEHH6zzqHtXxyRMGIWpgTeJh2E0GcnV5RPuf/VFWioW/shVq4l2g/7rCk6ePMmaNWvIyclh2rRpnDp1Cr1eT9u2bZ0tTSLxKOoUsF9++eVa36qNRqNNBTmKzad3sPH0X3ax3T/uNvrG9bzqMSaTiblz57JkyRI0Gg0ajYaoqChOnTrVYAK20WT+bTTsJvGKudjZ1wzYFQt/ZGqVRLd0/f5rgN9++42ZM2cyePBgEhISmDZtGiUlJbz33nt8/fXXzpYnkXgUdQrYzZs3v+r+Z555xiZiPIl9+/aRnZ3N+PHjLdtSUlLceinNK6kI2A26SdwyFzuH9tc4tiLLWW6Zhk5uMOAM4MMPP+Srr76iXbt2/PbbbwC0bdtWTuuSSJxAnZ6kzz77rL11OJy+cT2vWQu2J0eOHGHUqFG8+uqrAJw4cYLRo0djMpmYNm0aRqMRo9HI22+/7bZ9hgZLwG64Nexwv1AUKOo0Urwij3iJ8CYm3D2axHNzcy1N3xW/Q4VC4ba/SYnEnXH9US8NlLy8vCpzWdesWcPAgQNp27Ytr7/+Om+++SZarRatVutEldeHQTT8gK1WqQn1C65T8hTjpRp2icnbLaZ0AXTo0IGVK1dW2bZ69Wo6d+7sJEUSiefScNsqXZy4uDhWrFgBmGvXy5Yt4/vvvwdgx44dLF26lJCQELdOUGEwGQBQNdDEKRXUdS626VIe8RLhTVSoe9SwX3vtNSZMmMBPP/2EVqtlwoQJnD59mi+//NLZ0iQSj0MGbCcxZMgQVq1axYABAwgLC2Pu3LmWBDI9e/akZ8+evPHGGyQlJbntXHSDB/Rhg7kfe8+FgyQkbyA+vCWtw+JqPK6ihq30DXSppTSvRqtWrfjtt9/YuHEj/fr1IyYmhn79+uHv7x4tBBJJQ6LOT1Kj0cjHH3/MU089JVfpsQG+vr4sWLCg2vadO3eydu1ahBAYDAZat27tBHW2wTLoTNWwa9g3x3ZiX9oRvj3wEwD/6vs8naPbVTvOpCvChBL/wEBHS7wufH19ueuuu5wtQyLxeOocsFUqFYsXL+a5556zpx6P55ZbbuGWW25xtgyb4ClN4j2bdqVn067klxbyz/Wz+eHwL3SKalttYJZRW4RO4UNoI9fu5hgzZkydBpUtWrTIAWokEkkFVrVVjhgxgu+//56HHnrIXnokDQhPmNZVmWCfIO5vfxef7VnE3ouH6Na46ipXRl0RJSZvQoK8naSwbowcOdLZEiQSSQ1Y9SQ9dOgQCxcuZMGCBURHR1d5C5dv25Ir8YRpXVfSN+5WVh5bx5LDq+ga2wml4vJEDKO2kEKjF6FBPk5UeG3uvfdeZ0uQSCQ1YFXAHjVqFKNGjbKXFkkDo6JJ3JMCtlqpYlTHoXy44ysOpifSJaajZZ+hpJBik7fLB+zKCCFYunQpCQkJ5OXlsWrVKnbv3k1WVpbs15ZIHIxVAdvd37yFEA0m4YMQwtkSromnjBK/ks7R5pxnaUWZdKm0cqxRW0SJKYooNwrY8+bN46+//mL8+PFMnz4dgOjoaN5++20ZsCUSB2N14pRly5bx8MMPM2TIEB5++GG3WajDx8eHnJwctwh010IIQU5ODj4+rv3gN3hALvGaCPDyQ6VQkl9aaNkmhECUFVMifAgNdO3rVpkVK1bw6aefcvfdd1tedps0acL58+edrEwi8Tysqvr873//4+eff+bRRx8lNjaWixcv8sUXX5CZmclTTz1lL402oUmTJqSmppKVleVsKTbBx8eHJk2aOFvGVTEKz6xhKxVKGvkEVQnYpjItCmGiRLj+oLPKGI1Gy5zrioBdUlKCn597JH6RSBoSVj1Jly5dynfffUfjxo0t23r16sXYsWNdPmBrNBri4mpOaCGxD5endXleBtxgnyAKKgfsSmlJ3akPu2/fvrz99ttMnToVMLcUzJs3j/79+1tlp6Gv+y6ROAKrnqQ6na7aalLBwcGUlpbaVJSkYWAwViRO8awaNkCwbyPydAWWv42X0pKaNH54adyni+Cf//wnmZmZ3HzzzRQVFdGlSxcuXrzIP/7xjzrbqLzue0JCAkuXLiUzM9N+onNyoG9fOHPm8uf9+y9va6jU9L2t9YE1Nirvt6XO2uzV5TxXsVfTdlv8BoUVvPzyy+Lpp58WJ0+eFDqdTqSkpIhnn31W/OMf/7DGjF05f/68aNOmjTh//ryzpXg8G0/9JUb+8KTIKM52thSH879d34nHf37F8nfJiT3i5Kz7xMy3F9XbpjN/29nZ2eLgwYMiMzPTqvOMRqMYPHiwyMvLs2y77777xPbt2+ulo04+ePddIZRKIQYPvvy5TZvL2xoqNX1va31gjY3K+22pszZ7dTnPVezVtL0O/r/W79uqgF1UVCRefvll0bFjR9G2bVvRsWNH8fLLL4uCggJrzNgVGbBdh/UpW8TIH54UOSV5zpbicL4/tFKMWvKUMBqNQgghCg9uFCdn3Sdmf7yq3jad/dvetm2b+Oqrr8S+ffvqfM7u3btF165dxT333GP51759e5GcnFwvDdf0gckkROPGQoAQvr5ChIWZP1f88/MT4tdf61W2S1Pb97bGB9bYqLzNGp/WpYya7NXlPFexV3l/5e11+A1e6/dtVVtlQEAA//nPf5g9ezZ5eXmEhISgVHpe/6Skbnhi4pQKgn2CEEJQqC8m2CfIsvCHb1CIk5XVjRdffJFbb73VkvXs888/58MPPyQ+Pp4PPviAGTNmMGLEiGvaqW3d94KCAqZOnYperycoKIhp06bZRvjWrVBwqStCp4Mru+u0WpgwAU6dAhefZWEVtX1va3xgjY3K26zxaV3KqMleXc5zFXuV9+/adXl7BdfxG7Qq2vbo0cN8klJJWFiYJVjfeuutVhUq8Qw8dVoXmAM2QL7O3Hdt1BZiFAqCghs5U1ad2bdvHwMGDADAZDLx5Zdf8t5777Fs2TLmzZtX5+U1a1v3vXv37rz11lu8++67pKWlUVJSYhvhH3wAlW3VNI2zoABmz7ZNea7C1b53XX1grY3K2+rq07qWcaW9up7nKvYq9l+5vbbz64hVAbu8vLzGbSaTyeqCJQ0fT8slXpkQX3Ngzi81v12XFRWgFd6EuPjCHxUUFhYSFhYGQGJiImVlZdxxxx0A9OnTh4sXL9bJTlxcHHv37gUur/v+wgsvWPZv2rSJVq1a2W65zuTkmoNLZbRaWL3aNuW5CnX53pWpyQfW2riWvZqoaxlX2rsebc6wV7G/tnLq+Rus05O0YvUevV5fbeGP9PR0unTpYnXBkoaPJ6YmrcBSw740tau0qIAS4e02SVNCQkJITU2lSZMm7Ny5ky5duqC6tEyqVqu1fL4WV1v3ffny5Vy4cMGqEefX5OhR29lyJ2zxvR3hu/qWYWttrm6vFuoUsEeOHIkQgsOHD3P//fdbtisUCsLCwujZs6fdBErcl4omcaUHzsNudEXArsgjHusmSVNGjhzJxIkT6dWrFz///DP//ve/Lfv27NlDy5Yt62SntnXfN27cyAcffEC/fv2YNm0akydPrjZlVCKRVKVOAbsih/iNN95Iq1at7CpI0nAwCiNqpbrB5G+3Bh+1N75qH/IvzcU26YrQCm9CG7lHDfvJJ58kKiqKI0eO8NprrzF06FDLvtzcXB599NHrst+/f3+rk69IJJ6OVZ2L33//PXfddRddu3a1bNu3bx+//fYbr732ms3FSdwbg9HgkQPOKgiunJ60rJhiU5TbNImD+UW9pgV/3H0RIInEXbGqrTIhIYGOHTtW2daxY0cSEhJsKkrSMDAIo0f2X1cQ7NuIvNJChBCoDSXolb74eHveADyJRGIbrArYCoWi2mpXRqNRjhKX1IjBZPTIEeIVmGvYBQh9KUphQngHOFuSRCJxY6wK2N26deODDz6wBGiTycRHH31Et27d7CJO4t4YTAbUCg+uYV9qEjdemout8A10siKJROLOWFX9ee211ywjR2NjY0lLSyMiIoJPP/3UXvokbozR5OFN4j5B6MpL0RblAuDlH+RkRdZhNBp55JFHWLBgAV5eXs6WI5F4PFYF7OjoaFasWMHBgwdJT08nJiaGzp07y/SkkhoxmIweP+gMILcwAwDvwGAnqrEelUpFamqq7PKSSFwEqzsYlUqlTJQiqRNGD+/Drsh2lpWfSTjg1yjYqXrqwzPPPMOMGTN47rnniI6OrjJFT76oSySO5ZpP0zvvvJPffvsNMC9mX9uc2k2bNtlUmMT9MZgMqDz4oV5Rw84oyCEcCAxxj4U/KvOvf/0LgJUrV1q2CSFQKBQkJSU5S5ZE4pFcM2C/8cYbls/vvPOOXcVIGhZylLg5YOdp8zEJBcHh7hewN2zY4GwJEonkEtd8mlYeAV6xWpdEUhcqMp15KkHegSgUCvL1xWiFFyFBfs6WZDWNGzcGzDNCsrOziYyMdLIiicRzuebTdN68eXUyNGnSpDodd/r0aaZMmUJ+fj7BwcHMmTOHFi1aVDlm27ZtvP/++xw/fpxx48ZZ1tKVuBcGowEvtcbZMpyGUqkkyDuQolIdJcKbVkHuk+WsgsLCQmbOnMnatWtRq9UcOHCADRs2cOjQoSqrbkkkEvtzzQ7G9PR0y7+zZ88yf/58tm/fzrlz59ixYwfz58/n7NmzdS5w+vTpjBkzhrVr1zJmzJgaF65v2rQps2bNYsKECdZ9G4lLYfDwGjZAiE8QxUKPVvjg5+N+vpg+fToBAQH88ccfaDTml68uXbpYxrVIJBLHcc0nyNtvv235/MILL/Dee+8xZMgQy7Z169axZs2aOhWWk5NDYmIiX331FQBDhw7ljTfeIDc3t8pKPc2bNwfM/Wd6vb5u30TichhMRlQenDgFzP3Y6YpUytV+brkIyvbt29m6dSsajcaiPzQ0lJycHCcrk0g8D6uG8G7ZssWyiH0FAwcOZPPmzXU6Py0tjaioKMtauiqVisjISNLS0qyRIXETDCaDx9ewg30aoVWZMGn8nS2lXgQGBpKXl1dl28WLF4mIiHCSIonEc7EqYDdv3pxFixZV2bZ48WKaNWtmU1GShoGnZzoDaOQTSIlKgfBxz4A9cuRInn/+eXbs2IHJZGL//v28+uqrjB492tnSJBKPw6rqz6xZs3j22Wf54osviIqKIiMjA7VazUcffVSn82NiYsjIyMBoNKJSqTAajWRmZhITE1Mv8ZKqbDmzk3C/ENpHtnG2FEBmOgNopPHDqFBg8He/EeIAjz/+OF5eXrz++usYDAamTp3KAw88wPjx450tTSLxOKwK2O3bt2ft2rUcPHiQzMxMIiIiuOmmmyyDUa5FWFgY7dq1IyEhgeHDh5OQkEC7du2q9F9L6s93B5YRExjJ6wP/4WwpgMx0BuBnMr+wlPt5O1lJ/cjOzuaRRx7hkUceqbI9KytLNotLJA7mutJQde/enfLycrRabZ3PmTFjBgsXLmTIkCEsXLiQmTNnAuY3+cOHDwOwZ88e+vTpw1dffcUPP/xAnz592Lp16/VIbfAYTEYKyoo4kXOaUkNZnc8rLS+lRF/362edJs/OdAag0pnzcBv83HPxjMoDTCtz9913O1iJRCKxqvqTnJzMU089hZeXFxkZGdx1113s3r2bFStW8MEHH9TJRqtWrVi6dGm17fPnz7d87tatG1u2bLFGmseTX1oAgFGYOJZ1kpti2tfpvI92fk12SS6zB//TJqOYtdpSvLzUqNVqj890BqAsMc9yMPq6px+EENW2FRcXu+WId4nE3bHqKTJjxgyef/55RowYQffu3QFzLbsi37DEeeTpCiyfj2Qm1ylgG0xGDmUco8xQxpn8VOJCml63jqT3n6KkRW/6jH1UzsMGlMXm1g6Dj3u1NFSsG1BWVka/fv2q7MvPz5c1bInECVj1NE1JSWH48OEAljdsPz8/ysrq3gQrsQ8VAdvfy4+jGcl1Oudk7hnKLjWfbz2z87oDdqlOR4iikKKcc0DFtC73ClS2xlRUjAaBXm10thSreOeddxBC8MQTT/Cf//zHsl2hUBAWFkbLli2dqE4i8UysCtiNGzfmyJEjdOrUybLt0KFDclqXC5CrywfgtqY38/upbRTrSwjwuvpUoiOXAnu7iBvYdm43Y2+877qWTCzMNc/XVZUVYjKZEEJ4fA3bUFJIoLcJLe6VAKhi3YAdO3bg6+vrZDUSiQSsDNiTJk1i4sSJjB49mvLycj777DN++OGHKit6SZxDnq4ApULJbc26sf7kVpKyUuje+MarnnM08zjNGzXmztb9ef+v+RzOPMaN0XXr+66J4ksJNrwMxRiEuUbp6ZnOjNoi/NSCgrIiZ0upF59//nmt++q6foBEIrENVlWn+vfvz/z588nNzaV79+5cuHCBjz76iF69etlLn6SO5OkKCPFpRHxYS7xUGkvtuTbKjeUk55yiQ1Q8XWM74afxZeuZXdelQZtvDti+phIMJgOAx9ewKSvGz6gkv7TQ2UrqReW1BNLT0zl8+DBffvkl586dc7Y0icTjqPPT1Gg0MmTIEH799VdmzJhhR0mS+pBXmk+IbyPUKjXx4a1Iyjpx1eNP5Jym3FhOx8g2eKk03NKkC9vP70UIUe8RwLqiQvyAAEUppaWlAB6f6UxVXoKfUHHRTQN25bUEKtiyZQurV692ghqJxLOpcw1bpVKhUqnkADMXJVdXQIhvIwBiA6PI0uZe9fgjmcdRKBS0i2gNQOOgaEoNZVbN4b4Sfcnlkeq52RkAHp/pTGPU4YcXxfoSyo3lzpZjE3r16sXvv//ubBkSicdhVXvlww8/zOTJk5k4cSLR0dFVamJNm17/lCBJzeiN5XyzfynFei2Tbn0UpaL6e1aeroB2ETcAEOYXQoleS6mhDB91zRm2jmYeJy64Kf5e5pSZFQPUivUl+Grqt26zoeRyP21eVhbg2U3i+nIjvkKHvyocKKOgtIhwf/fK6nf+/Pkqf+t0OhISEmQ6YYnECVj1NK0YXPbnn39W2a5QKEhKSrKdKgkAJoOeU+u/YH7ZeU6X5QNwY3R7BrS8rcpxemM5xfoSQn2DASz/52rziA2Krma3zKDneM4p7mrd37It4FLgLtZrifAPq59e3eWAXZifDXh2k3heURn+yjIaaQKAPPJLC90uYA8aNAiFQmFJoOLr60u7du2YPXu2k5VJJJ6HVQH72LFj9tIhuQKjtpD0pXP4TKST5q3mmeZ9+F2XxqKDy+neuDOB3gGWY/MvzcEO8TE3iYf5hQCQo8uvMWAnZ5/EaDLSITLesq0iYJfoS+qtWZSVYBQKVAqBrtDcJO/RATu3AC+FkWC/YCg/b8lG507Ie14icR3q1Iet0+l4//33efLJJ/noo4/Q691rTqm7IIwGCnb/SsaK90n9/AV0aSe54OdDL+FP0y2/MDa2ByXlOr4/tLLKebkVAftSzTrs0v852qrrGFdwNPM4SoXS0oQOlZvE659XXKkvoUBpfmmo6M/25D7svBzzS0tEkLnFwl1HihsMBnbv3k1CQgJ79uzBYDA4W5JE4pHUqYb9+uuvc+TIEXr37s3atWvJz8/n3//+t721eRw5G76hcPevqIPC8W7cBrr0x7D3K1rfPBRNwQrUGxZz5+0D+fX4Rga0vJ0bwloA5hHiAKGXBp2FXqphVyRTuZKjmcdpFdq8Sl915T7s+qI2aClTB6LV6ygvLQQfz+7DLs7LJQqICouGnKrpY92FkydP8tRTT1FaWkpMTAxpaWl4e3vz6aef0qpVK2fLk0g8ijrVsLdu3cqCBQt45ZVXmD9/Phs3brS3Lo+jOPFPCnf/SlCPoTR77jOiR75Klq85oDYNa0Fw75EYCrK4J7g1wT5BzN+7GJPJvBJUnqWGbQ7YXioNgd4BNdawS8tLOZl7hg5XrJlduQ+7vniZdJg0fmgV/ohLgd+Tm8S1+fkABIeEE+gd4JY17JkzZzJq1Cg2b97MkiVL2LJlC6NHj5ZTOyUSJ1CngK3VaomMjAQgJiaG4uJiu4ryNPTZqWSt/gTvxvGEDRhr2X6xKB2A2KAo/Nt0R6H2wpi8m4e7/I3TeedZf9K85GiurgC1Ul0lFWmYbzA5NdSwj2WfxChMdKzUfw3gpfZCo1RfV8D2FqUIb3/K1AEoDGY77pbpbN3OsyzfmGITW/pi84uU2j+IYJ8gtwzYx44d4+9//3uVGSHjx4+XfdsSiROoU3ul0Whkx44dlpGiBoOhyt8At956q30U2phz6YUUlujp2Crc2VIAMOl1ZCx7B4Xai6j7XkKh0lj2pRam08gnyBKI/VrfTMmx7dw66O9siIznh8Mr6dm0C3k6c9KUyg/VUL8QcmuoYR/JTEalVBEfXr05M8DLv95N4uUGI36UUuQTiEGvR206D3jbvUn8bFohWw5c4KEhbVEqr2/JRyEE369LBiG4r/8N1z7hGpSXmAO00jfQbQN2ZGQku3btqnJ/79mzx/ICL5FIHEednqZhYWFMnTrV8ndwcHCVvxUKBRs2bLC9Ojvw++7zrNiUwvA+rXhkaHvUKuetJiWEICvhE8pzLhLz4L9RB1WdTnWxMIMmlUZ5B7TvTUnSdkrPHmHCzaP5x9pZLDy4gjxdAaGXRohXEOYbzInsU9XKPJpxnNahLfBWe1XbF+DlV++AXZRfgEohUPkGYigvR1NShjlg26+GrS83Mvvb3aRmFtP7psa0iAmqqkmrx8dLhUZdNw1n04vIztcBUFpmwMf7+l42Kqa5qXwDCPYJ4liWbWrujuSFF17g6aefpl+/fsTGxnLx4kU2bdrEO++842xpEonHUacn0h9//GFvHQ5j3J3tKDcYWbnlJMlnc3llXHciQpyzGlHh7tWUJP1FaP+x+MZ1rrJPCMGFwjRua9bNss33hi4ovP0oPvonjYc9w7D4O/g5aS0+am9uiu5Q5fwwvxCK9CXoDXq8LgVnrV7Hqfxz3Nfuzhr1BHj7U1LPJvHCioU/AoIwmATozP3r9gzYC9ccIzXT3D1z9FROlYBdUFzGs+9spHlMIG9MvK1O6Vb3JGVYPl/MLqFl40ZXOfraKPXFlGu8UajUhPg2Ir+08LpSvzqDgQMHsnz5cn777TcyMzNp3bo1zz//PHFxcc6WJpF4HB63WLFGrWTivZ15ZVw3zqYXMun9Tew7lulwHbpzieT8/g1+bXrQ6NYR1fYXlBVRUq6jcaUatlLthX98D7TJOxCGcu5rfyfhfqGUGsosA84qsCRPqdSPnZh1AiEEHaOq9l9X4K/xq3cfdsmlAVY+AUF4B4ViuhSU7NEkri0tZ/O+VH7enMKQns0JDfIm8XROlWM+X3GY/OIyDp7IZtvBi3WyuycpAz8fs94LWdc3TsNgNKE26DBqzN0ZwT5BlJsMaMt112XXGcTFxfH0008zY8YMnn76acrKynj++eedLUsi8Tg8LmBX0Pumxrw/uS+hQd7M+GI7i9cew2gS1z7RBhiK8shc/h6akCgihz1bY43rQqF5wFnjKxKfBLTvhalMi/bkfnzU3vy96ygAwvyCqxxXOXlKBUczj6NRqmkdVnPt6Hr6sHUF5nJ8GwXjFxKO8dJ2W8zDLtLq2XkkjQW/HOHFDzbz4L9/491Fe4kO8+fRYR1oHxdG4qkcy5iK7YfT2HLgAg8OjqdlbCO+/OUIpWVXnztcrCsn6Uwug3o0B64/YOcXleGvLEVcSnAT7GOu/btLP7ZOp+ODDz7gySef5O2336a4uJjz58/zzDPP8OCDDxIWVr9seBKJpP547iRZoElkIO9O6sOnyw/x/bpkkk7n8tJDNxMcWHP+bVsgjAYyV7yHSa8jZsx0lD7+NR5XW8D2bdEJpV8QxYnb8I/vQbfYzrzc60naXjGIrKbkKUczk2kTbl5+sybMfdj1q2GXXRoR7R8cgtLPgNFSw65/wE7LLuE/3+0mJdVsW6NW0qZZCCMHtKZ9yzDatwjFx1tNh5ZhbDt4kcw8HUH+Xvxv2UFaxjZi1B1tuLF1BFP+u42lf5xg3J3tai3rwPFMTCbB7Z1j+fPQxesO2LmFpfgr9Kh8zalIKwfsK6+pK/L666+TmJhIr1692LJlC8ePH+fUqVOMGDGCN954g9BQ90qxKpE0BDw6YAP4eKmZPLorHeLC+HT5ISa9v4lXH+5G+zj71CBy//iO0vNJRI6YjFdks1qPu1CYjrfamzDfkCrbFSo1/m17Unx4MyZ9KUovH7o3vrHa+VcmTykqK+ZMfiqjOg6rtcwAb3/KDGWUG8vR1BLUa6P80sIfgaGh+JgUGC81GtS3hl1WbmT2N7vJzNMy9s62dGwZTuumwXhpqtvr0NJ8rY6eyiG/qIy8ojKmPtIDtUpJh5Zh9OvahOUbU7ijezNiwmt+QdqTlEGgn4Y2zUNoHOHPxToE7NIyA3uOZbA/OQuD0cQz999o0WcO2KWo/c2BOvjSoEB3SZ6ydetWVq5cSVhYGOPGjaNfv34sXLiQbt26XftkiURiFzy2SfxKBt3SnHcn9cHbS8U/P/mTFZtSqkxbswXFiX9SsCuBoO53EdCh91WPvViUTuPAqBqbywM69EKUl6E9sbvW833U3vh7+ZF9aZnNxEvrY3e8ImFKZfw1FfnEra9lG7XmgO0b1IjA4EboL/20rOnDLjeYyMjVoi83Mv/nw5y6WMCLY7rywB3xdGgZVmOwBmgWHYS/j5r9xzNZsSmFm9pE0LbF5RrgI0Pbo1ErWPDLkZq1G03sScqgS5tIVEoFjSMCuJBZfM3r//qCncz5dg/bDl7gjz3nmbdkv+WcvMJS/JVl+ASaA3Wwr3s1iWu1Wkuzd3R0NH5+fjJYSyROxuNr2JWJi23E3Ml9+fDH/Xy56iiJp3OYNLorAb7W1TZrwqQvJWv1//BuEk/YwIer7Pvr3F6+P/QzKqUKfy8/WoU253TeeTpHt6/Rlk/TdqgCQyk++udVA3+Ybwi52nzAPJ3LW+XFDaEtaj0+wPtStrNyLcG+1o2QFmXFlAovFEoVCkCrMHcrWNMk/tGP+9m4N9Xy98iBrene/trNxyqlgrYtQtl06dzRg6oOqgtr5MsDd8Tz9epE9h7L4Oa2UVX2HzmVQ0Gxnts6xwLQOCKAklIDBcX6WrtHEk/ncPhkNuPubMff+t/A8k0pfPtrEjHh/oz9v3bk5ZcQrzDg18jc0uGv8UOtVLtNwL4y9wLgtrkXJJKGggzYV+Dvq2HKw935Zespvlp1lBfmbmLKw91p1ST4uuzqTh1A6HWE9n2wSnKUY1kn+Xjn1zQOjCI2KJqC0kI2nvqLMqOeuOCa1xhXKJQEtL+dgt2/YdQVo/INqPG4ML9gcnTmPuyjmcnEh7dCrar9klvyiZdZX8NWlJVQprycm7xM6QUI1HXMdJaaWcTmfanc3jmWuMZB+HipGXp73acOdWgZxt5jmXRsFWZpIq/MPX1asm7nWeb/fJjO/4hAo77cuLT1wAV8vFTc3M6cDCQ2wuzPC1nFtQbs5RtTCPTz4p7eLVGplNw/oDVp2SUsWX+cnh1jKCkw+72iSVyhUFxKnuIeTeINKfeCRNJQkAG7BhQKBcP7tCK+WQhzvt3Nyx9t5YkRnRjSs3md59AKIcgpKCU82DzHuyR5J0qfAHyaXa41ZxZn8+6fnxLhF8r0/i8Q4G0OmAaTkYuF6cQE1p5Nyr99Lwp2rqIkeSdBNw2s8ZhQ3xBO5JxhV+oBzhem0bvFLVfVXNsCICaTQKHgqt9dWV6CXnl5PrveKwAoqnNq0qUbTqBWq3jyvs71GvTXrV0U369L5qEhbWvcr1GreHxEJ2Z+sYNftpzkbwNaA+bm8L8OpdGjfTQ+XubboUnk5YBdU/A/n1HErsR0Hrgj3pJcRaFQMOGejmzef4G1O85iyDcHbKVfoOW8YJ8gCtykht2Qci9IJA0F2Yd9Fdq2COWDF/vRqVU4//3pIHO/33fN6UEVrNh0ksfeXE9OgQ5hNKBN2Ytf624oLjURa/U6Zm/9BKMw8Wqfpy3BGszNyM2CG1914Jd3TCvUIdGUJG6r9ZiogHCK9SW8++dnANxUSxN7Bf6WBUAuB2whBM++u5EvVx296rkaow7jpT5wAOEXiEIIdPm5Vz0PzKPBN+1L5a7bWtR7hH5cbCN+fOvuq6ac7dYuih7to1nyezI5Beb50IdSsinS6ul1U6zluIgQP9QqRY0Dz/TlRpasP45GpWRor6otAP6+GnrdGMvmfamWeekq30oB27cReTr3CNgSicT1kDXsa9AowJvpj/Xkxw3HWbz2GCcvFDDl4e40jQqs9ZwirZ4fNxzHaBIcP5cH5XsIKy0hIr4HAEaTkbnbvyCtKIPX+j5HbGBUrbZqQ6FQEND+dvL/WoGhOB91QHC1Y/7vhr40a9QYP40v4f4hhPtdfSpOxYpdB05eoE9zgVKpIDu/lPMZRVzIKmZQj2Y0iw6q8Vxvkw691+UWAVVAEKqSC6SfSOaGsIhayzSZBN/9loRKqeDefteXv7suaWYfG96RZ975g69XJ/LSmJvZeuACvt6qKv3aKqWCmHB/SxY1g9HEgeNZbD1wgR1H0tCWGrinT0saBVR/uRh8S3P+2HMenSkPAkHle9lfwT5BHM8+eV3fUSKReC4yYNcBpVLB6EHxtG0ewruL9vLiB5t5duRN9O3apMbjf9pwAm1pOQoF7D13jC3Fv9ErIohnW94EwDf7f+JgeiITuz1Ex6iam3DrQkCHXuT/uYySpL9o1P2uavt9ND50je1YZ3t+GnOT9uZDZ+jfLJsb20SQkppHmLKIEnxZsOooMx+vPtDIaBL4Uobe+3Jfuk9wMKoSQf75k9CzV43l6cuNzP1+H9sOXmTM4HhCg3xqPM6WxIT7c2+/G/jx9+McPJ5Fia6c2zrHVhuB3jgigJTz+Xy89AB/HbpIkbYcfx81t3eOpddNjbmxdc0vIe3jQmkaFYB/fhlgXvijgmCfIIrKSjCYjB697KhEIqkfMmBbwU1tIpn3Yj/+890e3l20l8TTOTw2vGOVxSWy83UkbDtFv65NOH4unxO5p8AL/mzkw6D8c5zNv8CalE0Mjb+Dga1qDmR1xSuiGV6RzShO3FZjwLYWpUKJWnhjUJdz9HQON7aJ4MS5HF4K+hVTUBTTjvVhT1IG3dpVbREoLinDT6mnuNLgN28/LxRCgSHrbI1l6cuNTJ+/nSMnc3h0WAdG9K2+epi9GD2oDcEB3py8kE9GrpZhvVtWO6ZZdBA7jqSzeV8qt3SIoU+XxnSJj7jmQiIKhYLBtzTn/Lq/AFD5XfZJsE8QAkFhWZEldaxEIpHUFRmwrSSskS9vPnU73/6axIpNKRw/n8+Uh7sTFeqHwWjio6UHMAnBQ//Xjm9WJ3JMf5pghRG1XxAf/LWA/LJCusV2Zmzne22ix799L/I2Laa8IBNNo+tf8tBkUIO6nKOnzLm5808fx19ZBsXnGB16mC9XBXFTm4gqzc85WdkAaPwvTwUTwoRCKPAqqp7HWwjBp8sPceRkDi+O6Ur/m2seDW8vNGpVjUG6Mvf2bUWHuDDatwy1DEarKwO7N2PTHiUYfarMCLBkO9MVyoAtkUisxuMGnZkMevSZ5zCWltQ7MYpapeTRYR147e89SMsqZvL7m9h5JI15P+xn37FMJt7bmahQP1rEBFLmnU9caTlPdRtLXmkBzRo15vmef0eptI3rA9rfDkBJ4l/XbSu3sBRDmRq1xkDyuTzKDSY02ccB8G9/O7dwkODcRNZuP1PlvONJ5n7Z6MaXa94GkxGFQkWgIQdhLK9y/JodZ1m/6xyj7mjj8GBdVwL8vOjaNtLqYA0Q6OfFLTcEWKZ0VVCxQIu7TO2SSCSuhcfVsHN//4bCvWsAUHj5oA4KRx0YhjooDFVQOOqgMPO2S5+VXrUvvdmzYwwfvNiPt7/ZzayvdgHm5Tv/79YWAIRHCspzTDQpb0SnZjfxuu8/aBwUhY/Gdn21mpBovKJbUXJsB8E1rPplDUmncxEGNcHBJtL0RrYfvkgzcZEyvyjihj3HhbwMxim28991kfTt2oQdaTvZcPJPOpwQ3ICCqA5dLbYMJgMqpQYVJrQZqfjHmkdUHzuTy+crDtG1bSRjapmC1RAwaouqjBAH91sARCKRuBYeF7BD+j6IT7P2GApzMBRmYyjMxliYgzbzLMaSAqBqrVvp428O5oGVg/nl/yODwnjnud4sXHMMf181Iwe2tpxbJs4DEOTVAoC2Efbppw1o15PcjYswFGShblT7iOxrkXgmhzDKUelzaanOIGHzCf6uyUTTtDcKtYao+15CP/8f/K38d2b+msVZo3mqV5TaQIF/C9QBl/OeG4QRtdo8ijrzZDJxsXHkFpby9je7CA/25eWHbkaldJ91oa3FpCtC6Vu1ht1IBmyJRHIdeFzAVvkGWJqRr0QYyzEU5WIozMFYKaBXBPeytBRM2uoPW6VfEIMDwwi8aSAKxeW0mOfSD6IxCbINraudY0v825oDdknyThr1GGrVuUaTQHkpKUriqRzi/PNJVin5W1AiP19U4t3IQFjbLgBogiMJGDqR9Tu+5KzxKP2b9uFk5gkSg1O5I7RnFbsGkxFvb1/KhZKy1FOUG0zM/mY3JaUGZj5xGwF+Xjb7/q6IUVeIJiy2yjYvlQZ/jS/5ci62RCKpBw4P2KdPn2bKlCnk5+cTHBzMnDlzaNGiRZVjjEYjs2bNYuvWrSgUCp544glGjhxpd20KlQZNcBSa4NrnRZsM+krB/PL/ZWknyVm7AO/oOHyamJt6T+SfJ7bMxOFcv1rt2QJNaCxekc0pTtp+zYCtLS0n+WweiadzSTqTQ/LZPCJD/XhlbDd8M48S0kyHTulPC87T38fczx7QshMAFwvTmXNqHZl+3tyfUYA+S8cNvgr+CFBR0qZxlXKMJiO+Xl6kG4MJyj7Hgl+OkHQml5fH3kyLmJrncjckjLriKlO6Kgj2aSRr2BKJpF44fNDZ9OnTGTNmDGvXrmXMmDFMmzat2jGrVq3i3LlzrFu3jiVLlvDRRx+RmppagzXHo1R7oQmNwbdFJwI79yOk1/1E3DWR2LEzUTcKJ3PVxxj0Osr0JaSKMsIJ5mx6MQajya66fNrcQmlqMjt3J2Ewmig3mPh91zneX7yX//50kP/+dJDJczfx4L9+Zdrn2/nx92SKSsrpf3NTcgtKefGDzQzwPoyPygehAJ2XD128z1LkFYnKL4iD6YlM/f0/lJRrmdZ/MvHEcHP+WrpdPI6fETanHaiix2AyoFGpyVFHEliQwg2HPuHF+NN0C8rCVI9c5e6EMJYjyrTV+rDBvGpXngcOOluyZAnDhw9nwIABzJs3z7licnKgb184c6bq52sdv39/zedZY68u59XXnsR1sNN1cWgNOycnh8TERL766isAhg4dyhtvvEFubi6hoZezcP3666+MHDkSpVJJaGgod9xxB2vWrOGxxx67bg2pBWnsuXjouu1ciRCCs23jOZyRROnyF4lReGFSKGgS2haD0cS3vyYR6Hf9q37VhK7MwIFdJp7VCM6t+oIDv0agUCjQlZUT4K1GCDAKwa1BPgzv6EdkiB8Rwb54aXKBXO4KKmPrrhRaKrLQtRgIeYfZdkNbvC8epywkggP7f+LXE3/QLCiWV3o/RYR/GLpxUzjx38mEKrW0V7djb9phlh391bL+dUZxFuF+oZwM601OqopOAblE5vxF+g9bQaHEKyoOn2btUPtbtyqYO2DSm5OmqPxqqmEHcTjjGD8nra22T6lQ0qfFLZbBaQ2FNWvWsHPnTn766SfKy8sZPHgwDz74IJGR1z8NsV58/TVs2wYTJ8LgwZc/r61+TaocP3o0pKRUP6+2zzXZq61sW9irTb/E8djpujg0YKelpREVFYVKZX6oq1QqIiMjSUtLqxKw09LSiI293P8XExNDenq6TTT8dX4PPx391Sa2riTIO4B4vwi8Mi9wKkDgr1Jx681DWLF9Pys2pdilzApuat0cAy25OecUcMa88cqWeB1wwfzvyizZtytA+IXS9sY7UG1JZKP+IoQHAFlwfAO3NOnCMz0etoxw9w0Jp7zXRE5v/YF7etzPkb2fsOTIqio2O0e3J6LtDazNVzPq2d4EegnKLhxHdy6R0vOJFO1bhzDobe4Ll0CpQhPRrNrmVqEt+PPcHhYf+rnaPgUKogLCuaVJFwcIdAwmk4m5c+eyZMkSNBoNGo2GqKgoTp065ZyALQTMnQsmE2zdCnv3mj9v2wa//QZ33ln78cfNUxyrnFfb55rs1Va2LezVpl/ieOx4XTxu0NnIDkMZ0e7/7GJbo1SjUCjMSUMUSoQQKBQKlsyKwWiq35zvuqBQmJOBCNETYazb4iQ12lGqUChVfPe3eZjE5SZ8BdS4EMnN/Xoj+vZCoVDwVbP3MIqqzf4V/hjRt5VlpS/fuM74xnUGQJiMCJOx3npdGYVCUSVpSgVD4wcy5IY+1PRrqM3P7sy+ffvIzs5m/Pjxlm0pKSlVXtAdytatUHCpS0Kng9JS82etFiZMgFOnwMen5uMrqHxebZ9rsldb2bawV5t+ieOx43VxaMCOiYkhIyMDo9GISqXCaDSSmZlJTExMteMuXrxI587mB/uVNe7rQaFQ4GXnh6JCobSUBaBSKVE5IHW0QqFEob7+0dfmPNd1E2z5jkoVqlrOqW1ZzooXBE+joQXlq3HkyBFGjRrFq6++CsCJEycYPXo0MTEx/POf/+Svv/5i8+bNjhP0wQdQUmn52MrJkwoKYPZsmDGj9uNrOq+2z1fau1rZtrBXk36J47HjdXHooLOwsDDatWtHQkICAAkJCbRr167a2/b//d//sXTpUkwmE7m5ufz+++8MGTLEkVIlEokNyMvLw9f3cvKhNWvWMHDgQAIDA3n77beJi4u7ytl2IDm5ahCsjFYLq1fX/fhrcaW967FVF3s16Zc4HjteF4c3ic+YMYMpU6bwySefEBQUxJw5cwB4/PHHef755+nUqRPDhw/n4MGDDB48GIBnnnmGpk1dM4WlRCKpnbi4OFasWAGYa9fLli3j+++/d56go1df1/26j3eULXvYk9gGO14XhwfsVq1asXTp0mrb58+fb/msUqmYOXOmI2VJJBI7MGTIEFatWsWAAQMICwtj7ty51brAJBJJ3Whwg86MRvMgJluNKpdIXIWK33TFb9wd8PX1ZcGCBTXumzlzJqdOnWLatGk8/vjjdWpFk/e3pCFzrXtcIeq7ZJWLsmfPHh566CFny5BI7MaiRYvo1q2bs2U4BXl/SzyB2u7xBhewS0tLOXLkCBEREZb53hJJQ8BoNJKVlUXHjh3x8dCpO/L+ljRkrnWPN7iALZFIJBJJQ8ThucQlEolEIpFYjwzYEolEIpG4ATJgSyQSiUTiBsiALZFIJBKJGyADtkQikUgkboAM2BKJRCKRuAEyYEskEolE4gbIgC2RSDwemY7Cvkj/2gYZsB1IYmIixcXFzpbhMkh/mJF+sB5b+6ysrMzy2R2Dy7lz5yi5tAazyWS6bnvSv67pU48J2Fu3bmXWrFlOeTDu2bOHMWPGsHjxYhQKhcPLrwnpDzPSD+6FrX22Y8cOnnjiCWbMmME333wDYNdrYevf286dOxk7diwzZ85k/PjxGI1GlMr6P9bd0b8e5VPRwDEajeLLL78UgwcPFu3btxcrVqxwWNl6vV688sorYujQoWL16tVV9plMJofpqIz0hxnpB/fC1j4rKysTs2bNEiNGjBC///67WL16tXjsscfEjz/+aCvJVbDH7+3o0aPivvvus/jj3nvvFYsWLaqXLXf0r6f5VAghGnwNW6lU0qRJExYtWsT777/PN998Q2ZmpkPK1mg0FBUV0aVLF+666y4Atm/fTkFBAQaDAXB885D0hxnpB/fC1j7TaDQ0bdqUDz/8kIEDBzJgwADi4uIoLS21i357/N527NhBy5YtueuuuzCZTDRp0oS2bdvWq6bpjv71NJ9CA138Y/Xq1RgMBtq3b0/r1q3R6/VoNBoUCgXjxo2je/fuPP/883Yp++zZszRv3hyTyYRSqSQ1NZVHH32UQYMGsWPHDsLDwwGIi4tjypQpCCHs3hwq/WFG+sG9sLXPNm7ciEajoXXr1kRFRVFQUEBQUBDl5eV4eXkxefJkunfvbrPlO239e7vSXnJyMvfeey9jxoxh48aNxMbGEhYWRnZ2NvPmzSMsLOyq9tzRv57m02rUq17uouTm5opHH31UPPjgg2L27NliyJAhYvfu3UKIy00Qe/fuFQMHDhSJiYlVzr3epsjk5GQxbNgw0aVLF3H27FkhhLnJRggh5s6dK+6//36RlJQkhBBi//79YtCgQeLQoUPXVea1kP4wI/3gXtjaZ4cPHxbjx48XDz/8sHjxxRfFhAkTxJkzZ6oco9frxcMPP2yT62/r31tN9v766y8hhBDnzp0Tc+bMqdIcPHz4cPHTTz/Vas8d/etpPq2NBtUknpGRQVhYGIsXL+bVV19l5MiRzJ49m7S0NBQKBUIIunbtSvfu3Vm6dCnp6el88sknwPUNhDAYDPzyyy+MHDmSnj17snjx4ipvTpMmTeLLL7+kbdu2ALRp04Y2bdpc/xe+BtIfZqQf3Adb+yw/P5+PP/6YQYMG8c033/DKK68QGxvL1q1bqxx38uRJANq1a0dmZiZLly4F6nf9bf17q8neu+++S3p6Ok2bNiUxMZHmzZtbjr/99tvJzc2t0Z67+teTfHo1GlTAPnHiBGfPngXMfQMTJkwgODiYX375BcDSb/DKK6+wePFi/va3v5GVlWU5vj4IIVCr1TzwwAOMGzeOl156iXXr1nHgwAHLBVMoFAQGBlrOWbhwIVqtlqZNm9b7u9YF6Q8z0g/ugT18FhwczOTJkxk5ciQAUVFRZGdnExoaCly+9ufPnycwMJAffviBiRMnWvpC63P9bf17q8leSEgIy5cvB6BTp06WUdILFizgjz/+oGfPntXsuLN/PcmnV8XqOrkLo9VqRb9+/SxNG0IIsWPHDtGrVy+h1+uFEEIcO3ZMjB07VjzxxBMiNTXVLjreeOMN8fTTT4vi4mJL80lZWZlYv369GDZsmHjxxRfFhQsX7FJ2ZaQ/zEg/uC+28FnFeRXX+rHHHhMbNmyocswrr7wi4uPjxfTp08W5c+euS7Otf2812du+fbvo3bu30Ov1oqysTDz33HPi0UcfFc8995xV+t3Fv57m09pwu4Cdk5NT4/YKx3z22Wfi0UcfrbJt9OjRlh9QRkaGOHz4sE3LrqCiz6KkpEQMHjxYrF+/XgghRF5enhBCiK1bt4o///yzXmVbiyP8cS0c6Y/CwsIatzvCD7WVXYEr/S5cBVvfS9eyV3HN09LSxPDhwy0P+Qp7q1evFps2bbL6e9RWjq1+b1ez98ADD4jff/9dCCFEeXm55bvUBVv/Jm3hX1vfw/WxdzWfusJ97jYBu6SkRMyYMUMMGzZMvPHGG2Ljxo1CCLOjy8vLLcelp6eLkSNHio8++kgIYb6RJ06cKNLT0+1ethCXL1pCQoIYMWKEmDRpkrj33ntFWVlZvcuviaKiIvH666+LnTt3Vttnb39crWyDwVDlb3v7o7i4WLz11lti/Pjx4r///a84cOCARYe9/XC1sh3tB3fB1veSNfaEEGL9+vXijTfeEEII8frrr4vJkydbgktdsPV950h79flNWmNPiPr519b3sCPtOfo+d5s+7Hnz5lFSUsL8+fNp3bo1r732GufPn0ehUKBWqwFYv349Wq2WGTNmsHnzZl566SUefPBBmjVrZhlOb6+y161bR2JioiUjTl5eHklJSYSHh/Pdd9/h5eV1/U64xNGjR3n88cdZvnw5X3zxBTqdrsp+e/rjWmWrVCrAMf7YunUro0aNwsfHh+eee46ioiI+//xziw57+uFaZTvSD+6Ere+luj4XDh8+DEBSUhJr1qzh/vvvR61W89Zbb6HRaOqk3db3naPtWfubrKu96/Gvre9hR9tz+H1uk7BvZ4qLi8Vzzz0njh07Ztk2ZcoUMXXqVKHVakVKSop44IEHxLPPPisyMzOFEOa3pa1bt4qUlBSHlV3RLLdt2zbx6quviuTk5OsquzbOnTtnaW4ZOnRotQw/KSkpYvTo0XbxhzVl29sfO3furFL+5s2bxfPPPy9yc3OrabG1H6wp21G/C1fH1veSNfaysrKEEEI899xz4qGHHqpXP6qt7ztn2quLf62xV1//2voedqY9R9znLhmwMzIyRElJSZVt48aNEx9//LHl7/T0dDF48GCxf/9+kZSUJLZv3+72ZVujqbi4WAghxI8//ijuvfdeyw0jhBB79uyxzFF057KvpaW0tFTodDpLM9Sff/4pxo4da9m/d+9eu/nBkWW7K7a+l2xhz5p+Xlv/9j3B3rX8a+v7yNXt2RqXCtgGg0H85z//EfHx8WLBggVCiMuDATZt2iSGDRtmebMRwjwab9KkSdVsuFvZ1miqiTFjxoh58+bVasPdyq6vls8++0zMmjXLplqcWba7Yut7ydH3pq1/+55mr7b9tryPXN2evXCpPuxt27ah0+mYMmUKq1atIjU11TKX7cYbbyQ+Pp7333/fcvztt99OVFQUJpPJMteuok/Bncq2RlNljEYjAJMnT2bt2rVcuHCBxYsXc+zYMcsxtvSHo8qur5bk5GRuvfVWABISEjhz5sx1a3Fm2e6Kre8lR9+btv7te5q9mrD1feTq9uyG3V8JrKCsrMzS9/HYY4+Jt956q0oauLNnz4pbb71VfPvtt2LDhg1ixIgR4rvvvnP7sq3RVBsPPPCAuPHGG8W999573f2zzi67PlpKSkrE448/Ln744Qfx5JNPiokTJ4rz58+7ddnuiq3vJUffm7b+7XuavfqWYc195Or27IVLBWwhLjd1JScni/79+4sDBw5Y+g+EMPchfPzxx+KBBx4QK1eubDBlW6OpYlt5ebnQ6XRi3rx5YsCAASIhIaHBlG2NFiHMfUnx8fFi3LhxDvWDvct2V2x9Lzn63rT1b9/T7FlbhhDW30eubs8eOCVgp6amitLSUiFEze3+FdtmzJghnn322QZTtr00Xc+gN2eWbUstWVlZ4rPPPnPLst0VW99Ljr43bf3b9zR7NWHr+8jV7Tkahy6vmZCQwOeff07jxo0pLS3lq6++AsBkMqFQKGpM0n733XfTtWtXNm3axKxZs+jbt6/blW0vTW+88Qb9+vVzu7JtreXNN9+kT58+ble2u2Lre8nR96atf/ueZs8ePr3yPnJ1e07DUW8Ga9euFaNHjxa7du0SJpNJ3HrrrWLPnj1VjklMTBTr16+3ZIVJT08XPXr0EKNGjbqumpwzy3ZFTa7kD+kH98LWPnP0NXB1/a5uT/rUuThslPjWrVsZOHAg3bt3JzMzk5tuuong4OCKlwbmzp3Lc889h0ajwcvLi6KiIj755BMmTZrEkiVLalwpxR3KdkVNruQP6Qf3wtY+c/Q1cHX9rm5P+tS52K1JPDU1lSZNmljWBf3555+ZPXs2AwYMYMeOHXTo0IHs7Gxat27NhAkTOHHiBH379q2Sxq7iXHcq2xU1uZI/pB/cC1v7zNHXwNX1u7o96VMXw9ZV9q1bt4oePXqIoUOHioKCgir7Tp48KV555RWxf/9+IYR5ZZe+ffuKQ4cOWY65nsnnzizbFTW5kj+kH9wLW/vM0dfA1fW7ur2acHXNnnCf27RJPD8/n3Xr1vHqq6/i4+NDQkJClf1hYWGcOnWKFi1aABAdHU3Hjh3JyMiwHFPfyefOLNsVNbmSP6Qf3Atb+8zR18DV9bu6vZpwdc0ec59fb8QvKyursmTZ8ePHhRBC/Pbbb+Kuu+6yLCReMZ9t0qRJYvLkySI/P1/Mnj1b3HffffVe4tCZZbuiJlfyh/SDe2Frnzn6Gri6fle3J33qHlxXH/Z3333H999/T7du3WjcuDETJ06s0vb/2GOP0bp1a1588UVL/8D58+d58803KSoqom3btrz44ov4+/u7VdmuqMmV/CH94F7Y2meOvgaurt/V7UmfuhH1jfTr168XI0eOFImJiWL//v3itttuE6tWrRJCXH6jOXr0qOjfv784efKkEEJYlh8rKSkR+fn59S3aqWW7oiZX8of0g3tha585+hq4un5Xtyd96l6orQnuxcXFBAQEAHDs2DHuuusu2rVrB8CUKVNYuHAhHTt2pEWLFhgMBtq3b8/dd9/N66+/jslkIiQkhHnz5uHn52f1i4Uzy3ZFTa7kD+kH98LWPnP0NXB1/a5uT/rUfanToDODwcDcuXN56qmnmDdvHseOHaNJkyasWrXKcsywYcMIDAzkl19+AS534BcUFLBv3z5uu+025s2bZ7VAZ5btippcyR/SD+6FrX3m6Gvg6vpd3Z70qftzzYC9f/9+/va3v1FSUsJLL72EyWTinXfeoXv37iiVSjZs2GA5duLEiSxbtgyj0YhCoeCnn35CoVCwadMmnnzySavFObNsV9TkSv6QfnAvbO0zR18DV9fv6vakTxsI12ozP3z4sPj+++8tf+/du1c8/fTT4ty5c+Lrr78W9957r2VfXl6eeP7550VGRoYQQlQZwVcfnFm2K2pyJX9IP7gXtvaZo6+Bq+t3dXs14eqa5X1enWvWsOPj47nnnnssC8H7+fmRkZFBbGws48aNw8/PjylTprBr1y5mzpyJEILw8HAA1GqrushdqmxX1ORK/pB+cC9s7TNHXwNX1+/q9qRPGwbXDNgajQY/Pz/LcPkjR44QFxeHSqVCqVQyd+5cbrjhBj799FOaNWvGhx9+iFJpm3wszizbFTW5kj+kH9wLW/vM0dfA1fW7uj131Czv8xqoa1W88jqhS5YsEUKYJ6gnJSUJIYRljVF74MyyXVGTK/lD+sG9sLXPHH0NXF2/q9tzR83yPr9MnV9HVCoVQghycnLQ6XS89NJL/PDDD5a3H29vb7u9VDizbFfU5Er+kH5wL2ztM0dfA1fX7+r23FGzvM8rYU10T0lJEfHx8WLkyJHixx9/tNlbg6uXXRvSH87X4kp+cBds7TNHXwNX1+/q9hxRhqvbc1esCthFRUXis88+syzy7UicWXZtSH84X4sr+cFdsLXPHH0NXF2/q9tzRBmubs9dsdt62BKJRCKRSGxHAx9SJ5FIJBJJw0AGbIlEIpFI3AAZsCUSiUQicQNkwJZIJBKJxA2QAVsikUgkEjdABmyJRCKRSNwAGbAlEolEInED/h9IEAojzYzwqQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 504x216 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed1000/pred.pkl')\n",
    "code = 'SH600157'\n",
    "date = '2018-09-28'\n",
    "lookbackperiod = 50\n",
    "\n",
    "prob = df.iloc[:, -3:].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
    "pred = df.loc[:,[\"score_0\",\"score_1\",\"score_2\",\"label\"]].loc(axis=0)[:, code].reset_index(level=1, drop=True).loc[date:].iloc[:lookbackperiod]\n",
    "e_all = pred.iloc[:,:-1].sub(pred.iloc[:,-1], axis=0).pow(2)\n",
    "e_all = e_all.sub(e_all.min(axis=1), axis=0)\n",
    "e_all.columns = [r'$\\theta_%d$'%d for d in range(1, 4)]\n",
    "prob = pd.Series(np.argmax(prob.values, axis=1), index=prob.index).rolling(7).mean().round()\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(7, 3))\n",
    "e_all.plot(ax=axes[0], xlabel='', rot=30)\n",
    "prob.plot(ax=axes[1], xlabel='', rot=30, color='red', linestyle='None', marker='^', markersize=5)\n",
    "plt.yticks(np.array([0, 1, 2]), e_all.columns.values)\n",
    "axes[0].set_ylabel('Predictor Loss')\n",
    "axes[1].set_ylabel('Router Selection')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('select.pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ2\n",
    "\n",
    "You could prepared the source data for this test as below:\n",
    "1. Random: Setting `src_info` = \"NONE\"\n",
    "2. LR: Setting `src_info` = \"LR\"\n",
    "3. TPE: Setting `src_info` = \"TPE\"\n",
    "4. LR+TPE: Setting `src_info` = \"LR_TPE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = {\n",
    "    'Random': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcNONE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
    "    'LR': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcLR_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
    "    'TPE': glob.glob('output/search/LSTM_Attn_tra/K10_traHs16_traSrcTPE_traLamb1.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl'),\n",
    "    'LR+TPE': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/pred.pkl')\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "910721fc4a7b46eea5ba6d50647320d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = {\n",
    "    name: backtest_multi(exps[name])\n",
    "    for name in tqdm(exps)\n",
    "}\n",
    "report = pd.DataFrame({\n",
    "    k: v[0]\n",
    "    for k, v in res.items()\n",
    "}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>IC</th>\n",
       "      <th>ICIR</th>\n",
       "      <th>AR</th>\n",
       "      <th>VR</th>\n",
       "      <th>SR</th>\n",
       "      <th>MDD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Random</th>\n",
       "      <td>0.159 (0.001)</td>\n",
       "      <td>0.321 (0.002)</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.362</td>\n",
       "      <td>11.4%</td>\n",
       "      <td>14.1%</td>\n",
       "      <td>0.810</td>\n",
       "      <td>21.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LR</th>\n",
       "      <td>0.158 (0.001)</td>\n",
       "      <td>0.320 (0.001)</td>\n",
       "      <td>0.053</td>\n",
       "      <td>0.409</td>\n",
       "      <td>10.3%</td>\n",
       "      <td>13.4%</td>\n",
       "      <td>0.772</td>\n",
       "      <td>20.8%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPE</th>\n",
       "      <td>0.158 (0.001)</td>\n",
       "      <td>0.321 (0.001)</td>\n",
       "      <td>0.049</td>\n",
       "      <td>0.381</td>\n",
       "      <td>10.3%</td>\n",
       "      <td>14.0%</td>\n",
       "      <td>0.741</td>\n",
       "      <td>21.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LR+TPE</th>\n",
       "      <td>0.157 (0.000)</td>\n",
       "      <td>0.318 (0.000)</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.460</td>\n",
       "      <td>12.4%</td>\n",
       "      <td>14.0%</td>\n",
       "      <td>0.885</td>\n",
       "      <td>20.4%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  MSE            MAE     IC   ICIR     AR     VR     SR    MDD\n",
       "Random  0.159 (0.001)  0.321 (0.002)  0.048  0.362  11.4%  14.1%  0.810  21.1%\n",
       "LR      0.158 (0.001)  0.320 (0.001)  0.053  0.409  10.3%  13.4%  0.772  20.8%\n",
       "TPE     0.158 (0.001)  0.321 (0.001)  0.049  0.381  10.3%  14.0%  0.741  21.2%\n",
       "LR+TPE  0.157 (0.000)  0.318 (0.000)  0.059  0.460  12.4%  14.0%  0.885  20.4%"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report\n",
    "# print(report.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ3\n",
    "\n",
    "Set `lamb` = 0 to obtain results without Optimal Transport(OT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUAAAAEDCAYAAABEXN1oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUjUlEQVR4nO3de0zV9/3H8RceRExBDcjlMEdtaWRk1EvXOGzqthQNpD30uMz2dNgtSye2GtalW1a1S7nMpM5s7pfW3iLZ7Cgmpdi1JzBvc6ZbaWi3NlvAHS/TYSzdERB+ririgcP5/UE4vyHYc5Dv8cj5PB9JEzl++J43X+3Tc/2cuEAgEBAAGGhatAcAgGghgACMRQABGIsAAjBWfLQHkKT+/n4dOXJEaWlpstls0R4HQIzw+/3q7u5Wfn6+EhMTx/z+TRHAI0eOaM2aNdEeA0CM2r17t+6+++4xl98UAUxLS5M0PGRmZmaUpwEQK86ePas1a9YEG3O1myKAI3d7MzMzNW/evChPAyDWXOuhNZ4EAWAsAgjAWAQQgLFuiscAgalqYGBAHR0d6u/vj/YoRrPZbJozZ47mzp2radPCv11HAIFJ6OjoUHJysubPn6+4uLhoj2OkQCCggYEBdXZ2qqOjQ9nZ2WF/L3eBgUno7+9Xamoq8YuiuLg4JSQk6Atf+IIuXbo0oe8lgMAkEb+bw0Tu+ga/JwJzAMCUwGOAmJJ8/gEl2KbfdMf2DfiVMN3697NbeVyn06n6+nolJibqtddeU0lJiVJTUyVJO3bsUF9fnzZu3Bh6Jp9Pv/rVr3To0CHFx8crMTFR5eXlWrFihd577z398pe/lCSdO3dOQ0NDSk9PlySVl5dr5cqVlvwsk0UAMSUl2Kbr4fr1ETn2m65Xrvt7E6bbVPJjt4XTDGvc7rTsWG73/89XW1ure+65JxjAiaiqqlJfX59+//vfa8aMGTpx4oTWrl2r2bNna/ny5Vq+fLmkiUX1RuMuMBAj3njjDVVXV0uSWltblZubq9bWVknDsaqvr5ck5ebm6tKlS3rllVfU1dWlJ598Uk6nUydPnpQkdXZ2qqysTMXFxVq3bp0uX7485ro+/fRT7du3T1VVVZoxY4YkacGCBXriiSf04osv3ogf1xIEEIgRy5YtU0tLiySppaVFS5Ys0QcffBD8etmyZaPWr1+/Xunp6XrhhRfkdrt1xx13SBrenWn79u3at2+fBgcH1djYOOa6Tpw4oezsbM2ZM2fU5YsXL9axY8ci8NNFBgEEYsStt96qK1eu6OzZs2ppadGPfvQjtbS0yOv1amBgIOzXx917772aNWuW4uLitHDhQp05c2bMmlj5LDUCCMSQgoICvfvuu+rp6dHSpUvV3d2td999V1/96lfDPsbIXVpp+B0Wfr9/zJoFCxbozJkzOn/+/KjL//73vys3N/e657/RCCAQQwoKCrRz504tWbJEknTXXXeppqZmzN3fEbfccosuXLgw4euZN2+eiouLVVVVpStXrkgavlv86quvqry8/Pp/gBuMZ4EBC/kG/JY+Y/vfxw3nZTAFBQV6+umng8ErKChQfX29CgoKxl3/3e9+V88884wSExO1ffv2Cc1UVVWl7du36/7779f06dM1Y8YM/fSnP9XSpUsndJxoirsZPhi9o6NDhYWF+uMf/8iGqAjbzfAymKNHjyovLy8ic2Dirv7zCNUW7gIDMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAgIV8/oEpdVzT8UJowEKR2qZrMlt0Xc2q/QDDdfV1XC2a+woSQMAwVu0HGK5Q1xHNfQXDugvc3t4ul8uloqIiuVwunT59esyanp4erVu3TiUlJcH3CA4ODlo2KIDPF+n9AC9duqTNmzfL4XDI4XBo586dweu+7777dOLEiTFfX+s6RkR7X8GwAlhZWanS0lIdOHBApaWlqqioGLPm1VdfVU5OjhobG9XY2Kh//OMfOnjwoOUDAxhfpPcDfPnllzU0NKTGxka98cYbcrvd+tOf/vS5M13rOkZEe1/BkAHs6emRx+ORw+GQJDkcDnk8HvX29o5aFxcXp0uXLmloaEg+n08DAwPKyMiIzNQAxoj0foAtLS166KGHFBcXp6SkJD3wwAPB4F6vaG9FEDKAXq9XGRkZstmGd6Kw2WxKT0+X1+sdtW7Dhg1qb2/XvffeG/zvK1/5SmSmBjCuSO4HGAgExnwE6MjXNptNQ0NDwctHtsgKJdr7Clr2Mpj9+/crNzdXzc3N+vOf/6yPPvpI+/fvt+rwAMIQyf0A77nnHu3Zs0eBQEAXL17U3r17g8fNzs5WW1ubpOFbiufOnQvrOqK9r2DIZ4Htdrs6Ozvl9/uD/xp0dXXJbrePWldXV6fnnntO06ZNU3Jysu677z59+OGHKi4ujtjwwM3G5x+w9CUr/33ccD6qM5L7AW7YsEFbtmxRSUmJJOnBBx/U1772NUnSD3/4Q23atEkNDQ266667lJWVdc3ruPpxwGjuKxjWfoDf+c53tHr1ajmdTrndbu3Zs0evv/76qDVPPPGE8vPzVV5eLp/Pp8cff1wrV65UaWlpyCHYDxDXg/0AcbWI7AdYVVWluro6FRUVqa6uLvhUe1lZWfBm7zPPPKOPP/5YJSUlWrVqlebPn6+HH37Yip8JACIirBdC5+TkqKGhYczlNTU1wV9nZ2dr165d1k0GABHGe4GBSYr2Szkw7L+fhQ4XAQQmITExUT09PUQwigKBgHw+nz799FPdcsstE/pe3gsMTMK8efPU0dGh7u7uaI9itPj4eM2ePVtz586d2PdFaB7ACNOnT9dtt90W7TFwnbgLDMBYBBCAsQggAGMRQADGIoAAjEUAARiLAAIwFgEEYCwCCMBYBBCAsQggAGMRQADGIoAAjEUAARiLAAIwFgEEYCwCCMBYBBCAsQggAGMRQADGIoAAjEUAARiLAAIwFgEEYCwCCMBYBBCAsQggAGMRQADGIoAAjEUAARiLAAIwFgEEYCwCCMBYBBCAscIKYHt7u1wul4qKiuRyuXT69Olx1+3du1clJSVyOBwqKSnRuXPnrJwVACwVH86iyspKlZaWyul0yu12q6KiQrW1taPWtLW16cUXX9Rvf/tbpaWl6cKFC0pISIjI0ABghZC3AHt6euTxeORwOCRJDodDHo9Hvb29o9a99tpreuyxx5SWliZJSk5O1owZMyIwMgBYI2QAvV6vMjIyZLPZJEk2m03p6enyer2j1p06dUqffPKJ1qxZo29+85t6+eWXFQgEIjM1AFggrLvA4fD7/Tp+/Lh27doln8+ntWvXKisrS6tWrbLqKgDAUiFvAdrtdnV2dsrv90saDl1XV5fsdvuodVlZWSouLlZCQoKSkpJUWFio1tbWyEwNABYIGcDU1FTl5eWpqalJktTU1KS8vDylpKSMWudwONTc3KxAIKCBgQF98MEH+tKXvhSZqQHAAmG9DKaqqkp1dXUqKipSXV2dqqurJUllZWVqa2uTJD3wwANKTU3V/fffr1WrVumOO+7Q6tWrIzc5AExSWI8B5uTkqKGhYczlNTU1wV9PmzZNmzdv1ubNm62bDgAiiHeCADAWAQRgLAIIwFgEEICxCCAAYxFAAMYigACMRQABGIsAAjAWAQRgLAIIwFgEEICxCCAAYxFAAMYigACMRQABGIsAAjAWAQRgLAIIwFgEEICxCCAAYxFAAMYigACMRQABGIsAAjAWAQRgLAIIwFgEEICxCCAAYxFAAMYigACMRQABGIsAAjAWAQRgLAIIwFgEEICxCCAAY4UVwPb2drlcLhUVFcnlcun06dPXXPuvf/1LixYt0rZt26yaEVOQb8Af7RGAkOLDWVRZWanS0lI5nU653W5VVFSotrZ2zDq/36/KykqtWLHC8kExtSRMt6nkx+6IHb9xuzNix4Y5Qt4C7OnpkcfjkcPhkCQ5HA55PB719vaOWbtz50594xvf0Pz58y0fFACsFjKAXq9XGRkZstlskiSbzab09HR5vd5R644dO6bm5mZ973vfi8igAGC1sO4ChzIwMKBnn31WW7duDYYSAG52IQNot9vV2dkpv98vm80mv9+vrq4u2e324Jru7m6dOXNG69atkyR99tlnCgQCunjxorZs2RK56QFgEkIGMDU1VXl5eWpqapLT6VRTU5Py8vKUkpISXJOVlaUPP/ww+PWOHTvU19enjRs3RmZqALBAWC+DqaqqUl1dnYqKilRXV6fq6mpJUllZmdra2iI6IABESliPAebk5KihoWHM5TU1NeOu/8EPfjC5qQDgBuCdIACMRQABGIsAAjAWAQRgLAIIwFgEEICxCCAAYxFAIEZEeg/GWNzj0ZLNEABEH3swThy3AAEYiwACMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBBGAsAgjAWAQQgLGMCCAfFwhgPEZ8LCYfFwhgPEbcAgSA8RBAAMYigACMRQABGIsAAjBWWM8Ct7e3a9OmTTp//rzmzJmjbdu2af78+aPWvPTSS9q7d69sNpvi4+P11FNPafny5ZGYGQAsEVYAKysrVVpaKqfTKbfbrYqKCtXW1o5as3DhQj322GOaOXOmjh07pkcffVTNzc1KTEyMyOAAMFkh7wL39PTI4/HI4XBIkhwOhzwej3p7e0etW758uWbOnClJys3NVSAQ0Pnz562fGAAsEjKAXq9XGRkZstlskiSbzab09HR5vd5rfs8777yj7OxsZWZmWjcpAFjM8neC/OUvf9Hzzz+v3/zmN1YfGgAsFfIWoN1uV2dnp/z+4fe7+v1+dXV1yW63j1n7t7/9TT/5yU/00ksv6fbbb7d+WgCwUMgApqamKi8vT01NTZKkpqYm5eXlKSUlZdS61tZWPfXUU3rhhRf05S9/OTLTAoCFwnodYFVVlerq6lRUVKS6ujpVV1dLksrKytTW1iZJqq6uVn9/vyoqKuR0OuV0OnX8+PHITQ4AkxTWY4A5OTlqaGgYc3lNTU3w12+99ZZ1UwHADcA7QQAYiwACMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBBGAsAgjAWAQQgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBtIDPPzAljw2YLj7aA8SCBNt0PVy/PiLHftP1SkSOC4BbgAAMRgABGIsAAjAWAQQQllh8so8nQQCEJRaf7OMWIABjEUAAxiKAAIxFAAEYiwACMFZYAWxvb5fL5VJRUZFcLpdOnz49Zo3f71d1dbVWrFihlStXqqGhwepZAcBSYQWwsrJSpaWlOnDggEpLS1VRUTFmTWNjo86cOaODBw+qvr5eO3bsUEdHh+UDA4BVQr4OsKenRx6PR7t27ZIkORwObdmyRb29vUpJSQmu27t3rx566CFNmzZNKSkpWrFihfbv36+1a9eGHMLv90uSzp49e70/R0gDfb0RO3ZHR4d8/3s5YseeqjjnNx7nfLSRpow05mohA+j1epWRkSGbzSZJstlsSk9Pl9frHRVAr9errKys4Nd2uz3soHV3d0uS1qxZE9b6m03h4Z9H7tj/UxixY09lnPMbbyqf8+7ubt16661jLr8p3gmSn5+v3bt3Ky0tLRhaAJgsv9+v7u5u5efnj/v7IQNot9vV2dkpv98vm80mv9+vrq4u2e32Mev+/e9/a+HChZLG3iL8PImJibr77rvDWgsAEzHeLb8RIZ8ESU1NVV5enpqamiRJTU1NysvLG3X3V5KKi4vV0NCgoaEh9fb26tChQyoqKprk6AAQOXGBQCAQatGpU6e0adMmffbZZ5o1a5a2bdum22+/XWVlZXryySd15513yu/362c/+5nef/99SVJZWZlcLlfEfwAAuF5hBRAAYhHvBAFgLAIIwFgEEICxCCAAYxFAAMYigACMRQABGIsAAjAWAZwEXkN+43HOb7xYPucEcII6Ojp07NgxSVJcXFyUpzHDeOd8aGgomiPFvPHO+bX21JvKeCtcmAYHB/Xzn/9chw8f1qxZs/Ttb39bq1evZvuuCBrvnH/rW99SfHx88PdHfg1rhPp7HmvnnFuAYWpra9OFCxd0+PBhbd26VXv27JHH45Ek7d69W++9956k2PxXMlrGO+cnT56UNLwD+bPPPqt33nlHUmzfTbuRxjvnx48flyQdOHBAGzduDH7eTyyccwIYwsgfckdHR3CvwytXrujUqVM6ePCgPvnkEx04cEA1NTWShnfMjoW/GNF0rXN+8uRJ7dmzR83NzWpra9Ojjz6qP/zhD/rPf/7DwxGT9Hnn3O126+OPP9aJEydUXl6uv/71r+rr64uJc85d4DA999xzmjlzpi5cuKB//vOfWrRokdxut/bt26ekpCT94he/0Be/+EU98sgjMXc3IVquPudLlizR7373O7355pvBzXZ//etf6/vf/35ww15MztXnfPHixXr77bf19ttvKy0tTY2NjTp06JAWLlyoBx98UGlpadEeeVK4BRjCyL8PBQUFqq2tVWZmpl5//XWtX79eixYtCn6eyYYNG7R//37t2rVLR44ciebIU961zvm6deu0ePHi4MMMR48e1eXLl+V2u3X06NFojjzlXeucP/7441q8eLF8Pp8uX76spUuX6vnnn9fRo0fV19cX5aknjwCGMHIzPzMzUw6HQ8uWLZMkdXV1qa2tTXPnzlUgENDFixd17tw5dXV1acGCBdEcecq71jnv7u5WW1ubZs+eLWn4ManDhw/rypUruu2226I2bywIdc6Tk5M1c+ZMnT17Vlu3blV+fr4yMzOjObIluJ8WpszMTCUlJcntdis3N1fnz59XYWGhkpOTJUlJSUnasWMH/yNa6FrnfNasWfL5fFqwYIEKCwt15513RnvUmPF557y/v199fX165JFHYubvOY8BTsBHH32kt956S62trRocHNTTTz+twsLRH+cXCAQUCAQ0bRo3rq0Q7jmXeF2mVcI559LweZ/q55wATtDg4KDef/99ff3rX4/2KMbgnN94ppxzAjgBV/+LxzOPkcc5v/FMOucEEICxeKAKgLEIIABjEUAAxiKAAIxFAAEYiwACMBYBBGAsAgjAWP8HimDX59TKOMMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb0.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
    "b = pd.read_pickle('output/search/finetune/Transformer_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_head4_hs64_bs512_do0.1_lr0.0005_seed3000/pred.pkl')\n",
    "a = a.iloc[:, -3:]\n",
    "b = b.iloc[:, -3:]\n",
    "b = np.eye(3)[b.values.argmax(axis=1)]\n",
    "a = np.eye(3)[a.values.argmax(axis=1)]\n",
    "\n",
    "res = pd.DataFrame({\n",
    "    'with OT': b.sum(axis=0) / b.sum(),\n",
    "    'without OT': a.sum(axis=0)/ a.sum()  \n",
    "},index=[r'$\\theta_1$',r'$\\theta_2$',r'$\\theta_3$'])\n",
    "res.plot.bar(rot=30, figsize=(5, 4), color=['b', 'g'])\n",
    "del a, b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ4\n",
    "\n",
    "You could prepared the source data for this test as below:\n",
    "1. K=1: which is exactly the alstm model\n",
    "2. K=3: Setting `num_states` = 3\n",
    "3. K=5: Setting `num_states` = 5\n",
    "4. K=10: Setting `num_states` = 10\n",
    "5. K=20: Setting `num_states` = 20\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = {\n",
    "    'K=1': glob.glob('output/search/LSTM_Attn/hs256_bs1024_do0.1_lr0.0002_seed*/info.json'),\n",
    "    'K=3': glob.glob('output/search/finetune/LSTM_Attn_tra/K3_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
    "    'K=5': glob.glob('output/search/finetune/LSTM_Attn_tra/K5_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
    "    'K=10': glob.glob('output/search/finetune/LSTM_Attn_tra/K10_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json'),\n",
    "    'K=20': glob.glob('output/search/finetune/LSTM_Attn_tra/K20_traHs16_traSrcLR_TPE_traLamb2.0_hs256_bs1024_do0.1_lr0.0001_seed*/info.json')\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = dict()\n",
    "for k, v in exps.items():\n",
    "    \n",
    "    tmp = dict()\n",
    "    for fname in v:\n",
    "        with open(fname) as f:\n",
    "            info = json.load(f)\n",
    "        tmp[fname] = (\n",
    "        {\n",
    "            \"IC\":info[\"metric\"][\"IC\"],\n",
    "            \"MSE\":info[\"metric\"][\"MSE\"]\n",
    "        })\n",
    "    tmp = pd.DataFrame(tmp).T\n",
    "    report[k] = tmp.mean()\n",
    "report = pd.DataFrame(report).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAADMCAYAAACoen5EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAn90lEQVR4nO3de1hTV7o/8C8JBEvVURQxFNTW1ooHqVREOeAVFEbB4AWxUscOrT4tKDO1tl56EabainPEtjjYo+P11DN2GDtaLketVKXY6lNGS1W8DcVRIAEEqXINJOv3hz/2SFGIEDYQvp/n4XlI9lr7XQvy8mavvdmxEkIIEBERyUDR0QMgIqLug0WHiIhkw6JDRESyYdEhIiLZsOgQEZFsWHSIiEg2LDoW5KuvvsLEiRPh4eGBnJycjh4OUZd269YthIeHw8PDAxs2bOjo4VgMFp02mDJlCr799lvpcXFxMdasWQNfX194eHggMDAQn3zyCaqqqmQZT1xcHN59912cO3cOI0aMkCUm0YNMmTIFbm5uKCsra/S8RqPBs88+i/z8fACATqfDsmXLMHbsWIwePRrBwcH44osvAAD5+fl49tln4eHh0egrLS1Nljl8/vnn6Nu3L86ePYtVq1bJErM7sO7oAViK8vJyzJ8/Hx4eHti/fz+cnZ2h1WqxY8cO3LhxA8OHD2+32PX19bC2tkZhYSGeeeaZVu3DYDBAqVSaeWTUnT3xxBNITU3FwoULAQBXrlxBTU1NozZvvvkmhg8fjuPHj0OlUuHq1asoKSlp1Ob777+HtbV8f6qEEBBCoLCwEEOHDoWVldUj76MhJ6kpHumYya5du/D444/jj3/8I5ydnQEAarUa77zzzgMLTsO7uM8//xy+vr7w9fXFzp07pe1GoxHbtm2Dv78/xo4di9/97ncoLy9v1DcpKQmTJk2SlgAMBgM0Gg38/f0BALm5uVi4cCE8PT0xY8YMpKenS/tftWoV1q5di8WLF2PUqFE4c+YMpkyZgj//+c8IDg7GqFGjsGbNGty6dQuvvPIKPDw88NJLL+Hnn3+W9hEdHQ0fHx+MHj0a4eHhuHbtWqP9x8bGYsmSJfDw8EBoaChu3Lghbb927Rp++9vfwsvLC//5n/+JTz/9tMV5U9ei0Whw8OBB6fHBgwcREhLSqM2FCxcwe/Zs2NnZwdraGiNGjMDEiRNbFW/hwoXYtGkT5s6di9GjR+O1115r9Nr54YcfMH/+fHh6emLmzJk4c+ZMo76bN2/G/Pnz8dxzz+Gtt97CwYMHsWPHDnh4eODbb7+FXq/H+vXrpXxdv3499Ho9AODMmTOYMGECtm3bBh8fH6xevRoJCQmIjo7GihUr4OHhgeDgYOTl5eG///u/4e3tjYkTJyIzM1Maw4EDB/DrX/8aHh4e8PPzw/79+6VtDfvfuXMnvL294evriwMHDkjba2pqsGHDBkyePBmjR4/GCy+8IBX45ubdIQS12uTJk8WpU6eEEEKEhoaKjz/+2OS+N2/eFMOGDROvv/66qKysFJcvXxZjx46V9rdr1y4RGhoqtFqtqK2tFe+++654/fXXG/V98803RWVlpaiurhZCCDFs2DBx/fp1IYQQer1e+Pv7i61bt4ra2lrx7bffilGjRonc3FwhhBArV64Uzz//vMjKyhIGg0HU1NSIyZMni9DQUFFSUiJ0Op0YN26cCAkJERcvXhS1tbVi4cKFIiEhQZpDUlKSuHv3rqitrRXr1q0TM2fOlLatXLlSjBkzRmRnZ4u6ujqxfPly8fvf/14IIcTdu3eFj4+P2LFjh6ipqRF3794VP/zwQ4vzpq6jITemTZsm/vnPf4r6+noxYcIEkZ+fL4YNGyZu3rwphBBi0aJFIiwsTKSkpIiCgoJG+2h4ndfV1ZkU88UXXxS+vr7iypUrorKyUixdulS88cYbQgghdDqd8PLyEidOnBAGg0FkZmYKLy8vUVpaKvWdOHGiuHr1qqirqxN6vV6sXLlSxMfHS/v/6KOPRGhoqLh165YoLS0VYWFhYvPmzUIIIU6fPi1cXV3Fxo0bRW1traiurhaffPKJcHNzExkZGaKurk68+eabYvLkySIxMVHo9Xrx+eefi8mTJ0v7P378uPjXv/4ljEajOHPmjHB3dxcXLlxotP+PPvpI6PV6ceLECeHu7i7Ky8uFEELExMSIF198Ueh0OlFfXy/+8Y9/iNra2hbn3RF4pGMm5eXlcHBweOR+UVFRsLOzw7PPPovZs2cjJSUFwL315Ndffx0DBw6ESqXC0qVLceTIEdTX10t9ly1bBjs7O/To0aPJfrOzs1FVVYUlS5ZApVLB29sbkydPRmpqqtTGz88Po0ePhkKhgK2tLQDgxRdfRP/+/eHo6AhPT0+4u7tjxIgRUKlUmDp1aqMLFObOnYuePXtCpVJh2bJluHz5Mu7evSttnzp1Ktzd3WFtbY2ZM2fi0qVLAIATJ06gf//+iIiIgK2tLXr27InnnnvO5HlT19FwtHPq1Ck89dRTcHR0bLT9448/hqenJxITE+Hn5weNRoMff/yxUZtx48bB09NT+srNzW023rBhw2BnZ4ff/e53OHz4MAwGAw4dOoQJEyZg4sSJUCgU8PHxgZubG06ePCn1nTVrFp555hlYW1vDxsamyb6Tk5MRFRWFfv36wd7eHlFRUfjyyy+l7QqFAtHR0VCpVFJOenp6Yvz48bC2tkZgYCBu376NJUuWwMbGBtOnT0dBQQHu3LkDAJg0aRIGDRoEKysreHl5wcfHB1lZWdL+ra2tERUVBRsbG0ycOBF2dnbIy8uD0WjEgQMH8Pbbb8PR0RFKpRLPP/88VCqVSfOWGxcdzaRPnz5N1qJNoVarpe+feOIJXL16FQBQWFiIqKgoKBT/fl+gUChQWloqPR44cOBD91tcXIyBAwc26u/k5ISioqIHxm7Qv39/6XtbW9tGj3v06CFdFGEwGLB582YcPnwYZWVlUpzbt2+jV69eTfZ1f1+tVotBgwY9cNzNzfuXf7Co89NoNHjxxReRn58PjUbTZPuvfvUrrFixAitWrEBZWRk2btyIqKgoZGRkSG1Onz5t8vmR+1/TTk5OqKurw+3bt1FYWIjDhw/j+PHj0vb6+nqMHTv2gX0fpLi4GE5OTo32X1xcLD3u27ev9OatQb9+/aTve/Togb59+0rnThsKU1VVFXr37o2TJ0/iT3/6E65fvw6j0YiamhoMGzZM6t+nT59GP4fHHnsMVVVVuH37Nmpra+Hi4tJkzKbMW24sOmbi7e2Nr776CkuXLm30B7MlWq0WQ4cOBXDvBTJgwAAA9wrKBx98gNGjRzfp03DlT3MnOAcMGACdTgej0SiNR6vVYsiQISaPrTnJyclIT0/Hrl274OzsjLt372LMmDEQJty0XK1WNzriul9z86au54knnoCzszNOnjyJ9evXN9vW3t4eERER+Pvf/97q83harbbR9zY2Nujbty/UajU0Gg3WrVv30L4tXTAwYMCARhfraLVaKV9N6d8cvV6P6OhoxMXFwc/PDzY2NoiMjDQpnxqK3c2bN5ucPzZl3nLj8pqZ/Pa3v0VlZSVWrlyJgoICAEBRURE+/PBDXL58+aH9EhMTUV1djWvXruGLL77A9OnTAQAvvPACPvroI2lfZWVlOHbsmMnjcXd3x2OPPYY///nPqKurw5kzZ/D1119L+2+ryspKqFQq9O3bF9XV1YiPjze576RJk3Dr1i3s3r0ber0eFRUVyM7OBtD2eVPns379euzZswd2dnZNtv3xj3/E1atXUV9fj4qKCvzlL3/B4MGD0bdv31bF+vLLL/HPf/4T1dXV+PjjjxEQEAClUomZM2fi+PHj+Oabb2AwGFBbW4szZ85Ap9OZvO8ZM2Zg69atKCsrQ1lZGf70pz8hODi4VeP8Jb1eD71eD3t7e1hbW+PkyZM4deqUSX0VCgXmzJmDDz/8EEVFRTAYDDh37hz0er1Z5m1uLDpm0qdPH/zlL3+BtbU15s2bBw8PDyxatAi9evXC4MGDH9rPy8sLU6dOxUsvvYSIiAj4+voCAH7zm99gypQpiIiIgIeHB+bNm9dkrbs5KpUKW7duRUZGBsaNG4fY2Fhs3LhROqpqq5CQEDg5OWH8+PGYMWMGRo0aZXLfnj17YufOnTh+/Dh8fHwQEBAgXVHT1nlT5zNo0CCMHDnygdtqamqwdOlSjBkzBv7+/igsLMTWrVsbtRkzZkyj/9PZtWvXQ2NpNBqsWrUKPj4+0Ov1ePvttwHce8efmJjY6MqxHTt2wGg0mjyPyMhIuLm5YebMmZg5cyb+4z/+A5GRkSb3b07Pnj3xzjvv4Pe//z3GjBmDlJQUTJkyxeT+K1euxLBhwzB37lx4eXnhv/7rv2A0Gs0yb3OzEqYcv5HZ5efnw8/PDxcvXuT1/ERmsHDhQsycOROhoaEdPRRqBo90iIhINiw6REQkGy6vERGRbHikQ0REsunSZ7Brampw4cIFODg48GaVZHYGgwElJSVwc3N74F0fuirmDbWnlvKmSxedCxcuIDw8vKOHQRZu37598PT07OhhmA3zhuTwsLzp0kWn4V5n+/bta/aWMEStodPpEB4e3qp76nVmzBtqTy3lTZcuOg1LAwMHDpQ+ToDI3CxtCYp5Q3J4WN7wQgIiIpINiw4REcmGRYeIiGTDokNERLJh0SEiItmw6BARkWxYdIiISDYsOkREJBsWHSIikg2LDhERycakopOXl4ewsDAEBAQgLCwM169fb9LGYDAgNjYW/v7+mDp1KpKSkhptT0tLQ3BwMIKCghAcHIxbt26Z1I+IiCyHSfdeW7t2LRYsWACNRoNDhw7hvffew969exu1SU5Oxo0bN3D06FGUl5cjJCQE3t7ecHZ2xvnz57Flyxbs2bMHDg4OuHv3LlQqVYv9iIjIsrR4pFNaWoqcnBwEBQUBAIKCgpCTk4OysrJG7dLS0hAaGgqFQgF7e3v4+/vj8OHDAIDdu3cjIiJCuutor169YGtr22I/IiKyLC0WHa1WC0dHR+mOoUqlEgMGDIBWq23SzsnJSXqsVquh0+kAALm5ubh58ybCw8Mxa9YsJCYmouFTspvrR9SVmbIsnZmZidmzZ8PNzQ1xcXGNtiUkJMDb2xsajQYajQaxsbHSttLSUixZsgTBwcEIDAxETEwM6uvr23tKRG0my0cbGAwGXLlyBbt27YJer8crr7wCJycnhISEyBGeqEOYsizt4uKCdevW4ciRI9Dr9U32ERISgpUrVzZ5/tNPP8XQoUOxbds21NXVYcGCBTh69CimT5/ebvMhMocWj3TUajWKiopgMBgA3CsgxcXFUKvVTdoVFhZKj7VarfQBUU5OTggMDIRKpULPnj3h5+eHH3/8scV+RF2VqcvSgwcPxogRI2Bt/Wjv/6ysrFBZWQmj0Qi9Xo+6ujo4OjqabfxE7aXFotOvXz+4uroiJSUFAJCSkgJXV1fY29s3ahcYGIikpCQYjUaUlZXh2LFjCAgIAHAv4TIzMyGEQF1dHU6fPo3hw4e32I+oqzJ1WbolqampCA4ORkREBM6dOyc9HxkZiby8PPj6+kpfo0ePNusciNqDSZdMx8TE4LPPPkNAQAA+++wzaW158eLFOH/+PABAo9HA2dkZ06ZNw7x58xAVFQUXFxcAwIwZM9CvXz9Mnz4dISEhePrppzF37twW+xF1Z/Pnz0d6ejqSk5Px8ssvIzIyErdv3wYAHD58GM8++ywyMzORkZGBrKwsXoBDXYJJx/RDhw594P/PbN++XfpeqVQ2OtF5P4VCgdWrV2P16tVNtjXXj7oWfZ0BKpvWfbRzW/p2RvcvSyuVyocuSzfn/s+Y9/HxgVqtxrVr1+Dl5YXPPvsMH3zwARQKBXr16oUpU6bgzJkzCAwMbI/pEJmNLBcSUPegslEi+I1DreqbvElj5tF0rPuXpTUazUOXpZtTVFQknae5dOkSCgoK8OSTTwIAnJ2dkZGRAXd3d+j1enz33XeYOnVqu8yFyJxYdCxMW48YuuIRR2c9woqJicGqVauQmJiI3r17S5dEL168GNHR0Rg5ciSysrKwfPlyVFRUQAiB1NRUrF+/HuPHj0d8fDwuXrwIhUIBGxsbbNy4UTr6WbNmDdauXYvg4GAYDAaMHTsW8+bNa5d5EJkTi46FacvRBtA1jzg66xGWKcvSnp6eyMjIeGD/X/7fzv0GDRqEXbt2tX2QRDLjDT+JiEg2LDpERCQbFh0iIpINiw4RNVFTX9Mhfcny8UICImqih3UPWMVataqvWCvMPBqyJDzSISIi2bDoEBGRbFh0iIhINiw6REQkGxYdIiKSDYsOERHJhkWHiIhkw6JDRESyYdEhIiLZsOgQEZFsWHSIiEg2LDpERCQbFh0iIpINiw4REcmGRYeIOg1+jo/l4+fpEFGnwc/xsXw80iEiItmw6BARkWxYdIiISDYsOkREJBsWHSIikg2LDhERyYZFh4iIZMOi0070dYYO6UtE1JmZ9M+heXl5WLVqFcrLy9GnTx/ExcVhyJAhjdoYDAasW7cO33zzDaysrLBkyRKEhoYCABISEvC///u/GDBgAADg+eefx9q1a1vc1pWpbJQIfuNQq/omb9KYeTRERJ2DSUVn7dq1WLBgATQaDQ4dOoT33nsPe/fubdQmOTkZN27cwNGjR1FeXo6QkBB4e3vD2dkZABASEoKVK1c+cP/NbSMiIsvR4vJaaWkpcnJyEBQUBAAICgpCTk4OysrKGrVLS0tDaGgoFAoF7O3t4e/vj8OHD7fPqIm6gLy8PISFhSEgIABhYWG4fv16kzaZmZmYPXs23NzcEBcX12hbQkICvL29odFooNFoEBsbK2176623pOc1Gg2GDx+O9PT09p4SUZu1eKSj1Wrh6OgIpVIJAFAqlRgwYAC0Wi3s7e0btXNycpIeq9Vq6HQ66XFqaioyMzPh4OCAZcuWwcPDw6RtRF2VKSsELi4uWLduHY4cOQK9Xt9kHw9bBdi4caP0/eXLl7Fo0SKMHz/e/JMgMjNZLiSYP38+0tPTkZycjJdffhmRkZG4fft2i9uIuipTVwgGDx6MESNGwNq69ffe/dvf/obg4GCoVKo2jZlIDi0WHbVajaKiIhgM966oMhgMKC4uhlqtbtKusLBQeqzVajFw4EAAgIODA2xsbAAAPj4+UKvVuHbtWovbiLqq5lYIHkVqaiqCg4MRERGBc+fONdmu1+uRnJyMOXPmmGXcRO2txaLTr18/uLq6IiUlBQCQkpICV1fXRktrABAYGIikpCQYjUaUlZXh2LFjCAgIAAAUFRVJ7S5duoSCggI8+eSTLW4j6s5MWQU4duwYnJyc4Orq2kGjJHo0Jh3Tx8TEYNWqVUhMTETv3r2lE56LFy9GdHQ0Ro4cCY1Gg+zsbEybNg0AEBUVBRcXFwBAfHw8Ll68CIVCARsbG2zcuBEODg4tbiPqqu5fIVAqlQ9dIWjO/Xlw/yqAl5eX9PyBAwd4lENdiklFZ+jQoUhKSmry/Pbt26XvlUplo6tr7vfLq3JM3UbUVd2/QqDRaB66QtCcoqIiODo6AnjwKoBOp8M//vEPbNq0yezjJ2ov/ORQonZiygpBVlYWli9fjoqKCgghkJqaivXr12P8+PEtrgL8/e9/x+TJk9GnT58OmiHRo2PRIWonpqwQeHp6IiMj44H9W1oFeO2119o2QKIOwHuvERGRbFh0iIhINiw6REQkGxYdIur2auprOrR/R2jLmNvSlxcSEFG318O6B6xirVrdX6wVre5bU1+DHtY9ZO/bljm3Zb4sOkREHaij/vh3FC6vERGRbCy+6PBjo4mIOg+LX17jx0YTEXUeFn+kQ0REnQeLDhERyYZFh4iIZMOiQ0REsmHRISIi2bDoEBGRbFh0iIhINiw6REQkGxYdIiKSDYsOERHJhkWHiIhkw6JDRESyYdEhIiLZsOgQEZFsWHSIiEg2LDpERCQbFh0iIpINiw4REcmGRYeIiGTDokNERLJh0SEiItmYVHTy8vIQFhaGgIAAhIWF4fr1603aGAwGxMbGwt/fH1OnTkVSUpK0LSEhAd7e3tBoNNBoNIiNjTWpHxERWRZrUxqtXbsWCxYsgEajwaFDh/Dee+9h7969jdokJyfjxo0bOHr0KMrLyxESEgJvb284OzsDAEJCQrBy5com+26pHxERWY4Wj3RKS0uRk5ODoKAgAEBQUBBycnJQVlbWqF1aWhpCQ0OhUChgb28Pf39/HD58uMUBtLYfUWdnygpBZmYmZs+eDTc3N8TFxTXa1twKAXAvd4KDgxEUFITg4GDcunWrPadDZBYtHulotVo4OjpCqVQCAJRKJQYMGACtVgt7e/tG7ZycnKTHarUaOp1OepyamorMzEw4ODhg2bJl8PDwMKkfUVdlygqBi4sL1q1bhyNHjkCv1zfZx8NWCM6fP48tW7Zgz549cHBwwN27d6FSqdptLkTmIsuFBPPnz0d6ejqSk5Px8ssvIzIyErdv35YjNFGHMHWFYPDgwRgxYgSsrU1a6Zbs3r0bERERcHBwAAD06tULtra25hk8UTtqseio1WoUFRXBYDAAuHfiv7i4GGq1ukm7wsJC6bFWq8XAgQMBAA4ODrCxsQEA+Pj4QK1W49q1ay32I+qqmlsheBSpqakIDg5GREQEzp07Jz2fm5uLmzdvIjw8HLNmzUJiYiKEEGadA1F7aLHo9OvXD66urkhJSQEApKSkwNXVtdHSGgAEBgYiKSkJRqMRZWVlOHbsGAICAgAARUVFUrtLly6hoKAATz75ZIv9iLqz5lYIDAYDrly5gl27duF//ud/kJGRgUOHDnXwiIlaZtIxfUxMDFatWoXExET07t1bOuG5ePFiREdHY+TIkdBoNMjOzsa0adMAAFFRUXBxcQEAxMfH4+LFi1AoFLCxscHGjRulZYHm+hF1VfevECiVyoeuEDSnIUeAxisEXl5ecHJyQmBgIFQqFVQqFfz8/PDjjz8iJCSkHWZDZD4mFZ2hQ4c+8P9ntm/fLn2vVCqbXF3T4JdX5dyvuX5EXdX9KwQajeahKwTNKSoqgqOjI4CmKwRBQUE4efIkNBoN6uvrcfr0aa4QUJfwaGcvichkpqwQZGVlYfny5aioqIAQAqmpqVi/fj3Gjx/f7ArBjBkzcOHCBUyfPh0KhQK+vr6YO3duR06XyCQsOkTtxJQVAk9PT2RkZDywf3MrBAqFAqtXr8bq1avbPlAiGfHea0REJBsWHSIikg2LDhERyYZFh4iIZMOiQ0REsmHRISIi2bDoEBGRbFh0iIhINiw6REQkGxYdIiKSDYsOERHJhkWHiIhkw6JDRESyYdEhIiLZsOgQEZFsWHSIiEg2LDpERCQbFh0iIpINiw4REcmGRYeIiGTDokNERLJh0SEiItmw6BARkWxYdIiISDYsOkREJBsWHSIikg2LDhERyYZFh4iIZMOiQ0REsmHRISIi2bDoEBGRbEwqOnl5eQgLC0NAQADCwsJw/fr1Jm0MBgNiY2Ph7++PqVOnIikpqUmbn376Cc899xzi4uKk5xISEuDt7Q2NRgONRoPY2NjWz4aoEzElbzIzMzF79my4ubk1ygug+dxg3lBXZW1Ko7Vr12LBggXQaDQ4dOgQ3nvvPezdu7dRm+TkZNy4cQNHjx5FeXk5QkJC4O3tDWdnZwD3itLatWvh7+/fZP8hISFYuXKlGaZD1HmYkjcuLi5Yt24djhw5Ar1e32QfzeUG84a6ohaLTmlpKXJycrBr1y4AQFBQEN5//32UlZXB3t5eapeWlobQ0FAoFArY29vD398fhw8fxiuvvAIA2LZtGyZNmoSqqipUVVWZZfAGgwEAoNPpmm1XV1XWqv3n5+e3ql9XjduRsTtj3IbXVcPr7FGYmjeDBw8GAKSnpz+w6LQHU/PGusKk96RNtPV32dXidmTszhi3pbxpMaJWq4WjoyOUSiUAQKlUYsCAAdBqtY2SR6vVwsnJSXqsVqul4JcvX0ZmZib27t2LxMTEJjFSU1ORmZkJBwcHLFu2DB4eHi0NCwBQUlICAAgPDzep/aPy+3pDu+y3s8btyNidOW5JSYlUHExlat60pLncaO+8eQpPmTzO+/l96deqfl01bkfG7sxxH5Y3rS/tJqqrq8O7776LDz/8UErA+82fPx+vvvoqbGxscOrUKURGRiItLQ19+/Ztcd9ubm7Yt28fHBwcHrhvorYwGAwoKSmBm5tbh8RvLjeYN9RZtZQ3LRYdtVqNoqIiGAwGKJVKGAwGFBcXQ61WN2lXWFgId3d3AP8+8ikpKcGNGzewZMkSAMCdO3cghEBFRQXef/99ODg4SPvw8fGBWq3GtWvX4OXl1eLkevToAU9PzxbbEbXWox7hNDA1b5rTXG4wb6gzay5vWrx6rV+/fnB1dUVKSgoAICUlBa6urk2WCAIDA5GUlASj0YiysjIcO3YMAQEBcHJywpkzZ/D111/j66+/xqJFizBv3jy8//77AICioiJpH5cuXUJBQQGefPLJVk2UqLMwNW+a01xuMG+oqzJpeS0mJgarVq1CYmIievfuLV3auXjxYkRHR2PkyJHQaDTIzs7GtGnTAABRUVFwcXFpcd/x8fG4ePEiFAoFbGxssHHjxkbv4oi6KlPyJisrC8uXL0dFRQWEEEhNTcX69esxfvz4ZnODeUNdlZUQQnT0IIiIqHvgHQmIiEg2LDpERCQbFh0iIpINiw4REcmGRYeIiGTDotPJdNTFhEajsUPiEpkD86brYNG5T21tbYfEvXPnDk6fPo3q6mpYWVnJFvfnn3/Gjh07UFdXB4WiY18K9fX13SquJWHedJyumDftfu+1rmLLli04fvw4oqOjMXHiRBiNRlleUDt37sTf/vY3DBkyBNbW1oiMjMTw4cNliZuWloZx48bB2toaQghZE/d+8fHxKC8vh4+PD8aPHw87OztZxtNRcS0J84Z586hxu33RMRgM2L9/P9LT0zFq1CikpaXB29sbKpWqXX+BNTU1SExMxLVr17Bjxw6o1WpoNBoUFRVh+PDh7Ra7srISGzZswPHjx3Ho0CH069fP7DFMdefOHbzzzjtQqVSYNGkSPvvsM2RnZ+PVV19F7969LS6uJWHeMG9aG7fbFx2lUonx48fD398fJSUl2LlzJw4ePIh58+a1a1yVSoV58+ZJH3KXm5sLOzs7lJaWoqqqCnZ2du0S19bWFs899xyqqqrw+OOPQ6vVIj09HUOGDIG3t7esdx2uqqpCfn4+vvjiCwCAk5MTDhw4gH379uG1116zuLiWhHnDvGlt3G59Tqfh5KOTkxMcHR3x9NNPw9PTExkZGdDpdLCysmqXE5RCCCgUCilxzp49i6ioKIwePRonTpxAXFwcvvvuO7PHNRgMsLa2xtixY2Fvb49f//rXWLp0KcrKyvCHP/wBmzdvRkFBgdnjNvjl2r/RaMSgQYNw9uxZAMDIkSMxYcIE/PDDD/jpp58AmOcEcUVFBfLz86UPlTIajXBxcWn3uJaKecO8aUvcbll0Gq44aTgMt7a+d8DXcMv3Pn364K9//avUxlwJ9Mu4DZ5++mkcPHgQK1aswJo1a1BTU4ObN2+aJeb9cRvejbm4uMDX1xcajQZ79uxBdHQ0Nm3ahO+//x4VFRVmi3u/LVu2YMGCBTh58qT0XI8ePWA0GpGXl4fq6mrY2Nhg+PDhcHJykl7EbV0q2bFjB+bMmYN169ZhxYoV0Ov10hp0e8a1RMwb5o058qbbFB0hBIQQjU50njx5Env27EFlZaXUbsiQIfD398f169exfft2vP/++/jXv/7VrnF79+6NHj16AAAGDhyImpqaNt8x+GFxd+/eDb1ej3HjxmHZsmXo2bMngHvvWlQqldmvRDIYDNi3bx/S09Ph7u6OtLQ06WOZ7e3tMXbsWJw6dQrXrl0DcO9zOPLy8qSEb+0frobPazp79ix2796NzZs3Izc3F1999RX69OkDT0/PdolraZg3zBtz541FFx2dTof9+/cDgHSCUaFQ4ObNm3jjjTewbds2eHp6Si9c4N6asa2tLb7//nvs378fnp6eGDJkSLvFvf+XlJ+fj7feegvl5eV45pln2mW+Y8aMgVKphK2trfQOrrCwEG+88QYef/zxR55rSxrW/j/99FPMmTMHdXV1OHjwoLQ9LCwMjz/+OJKSkpCVlYWKigoYDAb06tULQOvfsalUKnh6euKTTz6BWq3GY489Bl9fX5w6dQoAsGDBgnaJawmYN8ybds0bYcHefPNNsXnzZulxXV2d2LRpk5g1a5b48ssvhRBC7Ny5U3zwwQfi7t27QgghdDqdmDp1qvjkk09kiVtRUSFqa2vFX//6VxEcHCy2b98uS9y7d+8KvV4v9u3b1+a4D2M0GqVxCCFEdXW12Ldvn4iKihI6nU5ql5+fL3bv3i0WLlwopk2b1uaxGAwGIYSQfqf19fVCCCGio6NFamqq1K6goMCscS0F84Z5I0T75Y3FFZ2ffvpJGAwGcefOHTFr1iyRnZ0thBDi/PnzIjw8XMTHxwu9Xi9OnTolXnjhBbF69WpRWlraaB+VlZWyx/3pp5/EnTt3ZI+bk5Mjfv7550eO25yGF++DXLlyRbz99tvi448/lp5rSLL8/HxRU1Nj9rgNyfPSSy+JK1euNNne1riWgHnDvPml9sobiyo6JSUlwtvbW2i1WnH27Fnx8ssvi9raWiHEvR9QaWmpuHHjhoiOjhYRERHSC02I5n/h7Rm34RfbleI+iNFoFEajsdHP8cSJE2L37t2ioqJCeq62tlYcP35cvP7662Lbtm3iD3/4g8jNzW2XuPf/EdTpdCI8PFwIIURmZqZISEgQxcXFrY5rSZg3zJv747Z33ljU/+n0798fGo0GqampqK6uxqhRo6BSqVBfX48nnngCWVlZ2Lp1K4KCgjBr1iwA/z5x2Jb/om5L3LZc399RcRvodDqcOHEC8+fPl36GVlZWuHnzJj766CPodDqsWbPmoWv/2dnZWLFiBZ566ql2iWtrayut0X/zzTeoq6vD22+/jatXr+K1117jxzv/f8wb5o2sedOmktUJVVdXiwkTJogxY8aI3/zmN+L//u//HnhoKIR537V0t7hCdI21/4allw8//FCMHTtW7Nmzp9VxLVl3e/0ybzoub6yEsLxrQ48dO4YNGzYgJCQEubm5uHLlClQqFcaNGwd3d3dMnz6dcVspLy8PgwcPRmVlJRYtWoSYmBi4u7vjwoUL2LBhA0aPHo2lS5fi+++/x5YtWzBkyBCsWLEC9vb20j5a85/jbY373XffYcSIEfjVr35ltp+FpekOr9+Oisu8uY/ZylcnYjQaxcyZM8W3334rhBCiuLhYfPXVVyI+Pl5kZWUxbit1xbX/hiuAqGWW/vrtqLjMm8Ys8kgHAC5cuICYmBhs374dffv2ZVwziYuLQ//+/VFdXQ0AWLp0Kerr62Ftbd1ua/8dGbe7sfTXb0fFZd78m8UWHQB49dVX8corr8DT05NxzaSmpgYBAQGorq6Gq6srXnjhBTz11FMYNmxYk7YGg8FsN0LsqLjdkSW/fjsqLvPm3yy66HTUHx9Lj9sd1uC7M0t//XZUXObNPRZ1yfQvddS7XUuP6+fnh4SEBOkkZElJCbKzs3H+/Hk4OjpaXNzuxtJfvx0Vl3nz/7Xb2SKyaOfPnxdz5swRZWVl3SIukTkwb4TgWVZqFTc3N/Tv3x+5ubndIi6ROTBvLPycDrUvS1+DJ2oP3T1vWHSIiEg2XF4jIiLZsOgQEZFsWHSIiEg2LDpERCQbFh0iIpINiw4REcnm/wHMDWmf8u7bhwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(6,3)); axes = axes.flatten()\n",
    "report['IC'].plot.bar(rot=30, ax=axes[0])\n",
    "axes[0].set_ylim(0.045, 0.062)\n",
    "axes[0].set_title('IC performance')\n",
    "report['MSE'].astype(float).plot.bar(rot=30, ax=axes[1], color='green')\n",
    "axes[1].set_ylim(0.155, 0.1585)\n",
    "axes[1].set_title('MSE performance')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('sensitivity.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IC</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>K=1</th>\n",
       "      <td>0.053247</td>\n",
       "      <td>0.157792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K=3</th>\n",
       "      <td>0.055535</td>\n",
       "      <td>0.157410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K=5</th>\n",
       "      <td>0.059224</td>\n",
       "      <td>0.156796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K=10</th>\n",
       "      <td>0.059403</td>\n",
       "      <td>0.156766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K=20</th>\n",
       "      <td>0.059193</td>\n",
       "      <td>0.156801</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            IC       MSE\n",
       "K=1   0.053247  0.157792\n",
       "K=3   0.055535  0.157410\n",
       "K=5   0.059224  0.156796\n",
       "K=10  0.059403  0.156766\n",
       "K=20  0.059193  0.156801"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9de784e21d4a351f53a5792b09a6ae66a23802b850ad98f62e10c0156e418c04"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": ""
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}