GB - Results of hyperparameter search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper param search results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd = pd.read_csv(\"Hyperparam_search/Graident_Tree_Boosting_GS_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_n_iter_no_change</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "      <th>...</th>\n",
       "      <th>std_test_accuracy</th>\n",
       "      <th>rank_test_accuracy</th>\n",
       "      <th>split0_test_F1</th>\n",
       "      <th>split1_test_F1</th>\n",
       "      <th>split2_test_F1</th>\n",
       "      <th>split3_test_F1</th>\n",
       "      <th>split4_test_F1</th>\n",
       "      <th>mean_test_F1</th>\n",
       "      <th>std_test_F1</th>\n",
       "      <th>rank_test_F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1363.824692</td>\n",
       "      <td>160.471544</td>\n",
       "      <td>4.911603</td>\n",
       "      <td>0.592267</td>\n",
       "      <td>0.10</td>\n",
       "      <td>500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 500, 'n...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003503</td>\n",
       "      <td>1</td>\n",
       "      <td>0.801555</td>\n",
       "      <td>0.809922</td>\n",
       "      <td>0.798996</td>\n",
       "      <td>0.811879</td>\n",
       "      <td>0.797032</td>\n",
       "      <td>0.803877</td>\n",
       "      <td>0.005944</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1192.315114</td>\n",
       "      <td>131.051183</td>\n",
       "      <td>4.252643</td>\n",
       "      <td>0.439848</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1000, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002783</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802123</td>\n",
       "      <td>0.805359</td>\n",
       "      <td>0.798023</td>\n",
       "      <td>0.812596</td>\n",
       "      <td>0.794693</td>\n",
       "      <td>0.802559</td>\n",
       "      <td>0.006184</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1172.526933</td>\n",
       "      <td>491.750465</td>\n",
       "      <td>4.173367</td>\n",
       "      <td>1.614136</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1500, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006741</td>\n",
       "      <td>3</td>\n",
       "      <td>0.787281</td>\n",
       "      <td>0.815247</td>\n",
       "      <td>0.800108</td>\n",
       "      <td>0.806662</td>\n",
       "      <td>0.790726</td>\n",
       "      <td>0.800005</td>\n",
       "      <td>0.010244</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>3434.386611</td>\n",
       "      <td>64.535056</td>\n",
       "      <td>5.118638</td>\n",
       "      <td>2.428189</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1500, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003079</td>\n",
       "      <td>4</td>\n",
       "      <td>0.787937</td>\n",
       "      <td>0.799485</td>\n",
       "      <td>0.788407</td>\n",
       "      <td>0.796244</td>\n",
       "      <td>0.781238</td>\n",
       "      <td>0.790662</td>\n",
       "      <td>0.006486</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>3049.183289</td>\n",
       "      <td>113.677140</td>\n",
       "      <td>9.204574</td>\n",
       "      <td>1.622832</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1000, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>5</td>\n",
       "      <td>0.778441</td>\n",
       "      <td>0.789974</td>\n",
       "      <td>0.779116</td>\n",
       "      <td>0.790491</td>\n",
       "      <td>0.774551</td>\n",
       "      <td>0.782515</td>\n",
       "      <td>0.006494</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "1           1    1363.824692    160.471544         4.911603        0.592267   \n",
       "2           2    1192.315114    131.051183         4.252643        0.439848   \n",
       "3           3    1172.526933    491.750465         4.173367        1.614136   \n",
       "7           7    3434.386611     64.535056         5.118638        2.428189   \n",
       "6           6    3049.183289    113.677140         9.204574        1.622832   \n",
       "\n",
       "   param_learning_rate  param_n_estimators  param_n_iter_no_change  \\\n",
       "1                 0.10                 500                      20   \n",
       "2                 0.10                1000                      20   \n",
       "3                 0.10                1500                      20   \n",
       "7                 0.01                1500                      20   \n",
       "6                 0.01                1000                      20   \n",
       "\n",
       "   param_subsample                                             params  ...  \\\n",
       "1              0.7  {'learning_rate': 0.1, 'n_estimators': 500, 'n...  ...   \n",
       "2              0.7  {'learning_rate': 0.1, 'n_estimators': 1000, '...  ...   \n",
       "3              0.7  {'learning_rate': 0.1, 'n_estimators': 1500, '...  ...   \n",
       "7              0.7  {'learning_rate': 0.01, 'n_estimators': 1500, ...  ...   \n",
       "6              0.7  {'learning_rate': 0.01, 'n_estimators': 1000, ...  ...   \n",
       "\n",
       "   std_test_accuracy  rank_test_accuracy  split0_test_F1  split1_test_F1  \\\n",
       "1           0.003503                   1        0.801555        0.809922   \n",
       "2           0.002783                   2        0.802123        0.805359   \n",
       "3           0.006741                   3        0.787281        0.815247   \n",
       "7           0.003079                   4        0.787937        0.799485   \n",
       "6           0.002896                   5        0.778441        0.789974   \n",
       "\n",
       "   split2_test_F1  split3_test_F1  split4_test_F1  mean_test_F1  std_test_F1  \\\n",
       "1        0.798996        0.811879        0.797032      0.803877     0.005944   \n",
       "2        0.798023        0.812596        0.794693      0.802559     0.006184   \n",
       "3        0.800108        0.806662        0.790726      0.800005     0.010244   \n",
       "7        0.788407        0.796244        0.781238      0.790662     0.006486   \n",
       "6        0.779116        0.790491        0.774551      0.782515     0.006494   \n",
       "\n",
       "   rank_test_F1  \n",
       "1             1  \n",
       "2             2  \n",
       "3             3  \n",
       "7             4  \n",
       "6             5  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.sort_values(by=[\"mean_test_F1\"], ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01, 0.1 ])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(dd[\"param_learning_rate\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 100,  500, 1000, 1500])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(dd[\"param_n_estimators\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR,NE = np.meshgrid(np.unique(dd[\"param_learning_rate\"].values), np.unique(dd[\"param_n_estimators\"].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.01, 0.1 ],\n",
       "       [0.01, 0.1 ],\n",
       "       [0.01, 0.1 ],\n",
       "       [0.01, 0.1 ]])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd_grid = lambda df,lr,ne: df[df[\"param_learning_rate\"]==lr][df[\"param_n_estimators\"]==ne][\"mean_test_F1\"].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Ghosty/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "Y_1 = [dd_grid(dd,0.1, n) for n in np.unique(dd[\"param_n_estimators\"].values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Ghosty/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "Y_2 = [dd_grid(dd,0.01, n) for n in np.unique(dd[\"param_n_estimators\"].values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z = np.vstack((Y_2, Y_1)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAGbCAYAAAAryfFSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de7yddX3g+89XtgFFlHCR0YTI3jVe0Fq0FK1O1eqo6PQlPeOliT2KHSpzptLjaLVg26MUtWMvU7QWL2ll1I5DpNRLeppKOV6q9aASFdRAwLCjIcLI3XojkPidP55nwZOVfV+X57I+79drv7LXs5619i+vbbYfvr9n7RWZiSRJ0qS4X90LkCRJGifjR5IkTRTjR5IkTRTjR5IkTRTjR5IkTZSpuhewkMMOOyyPOOKIupdRm7vvvrvuJUiSxujuu+/mrrvuujUzjx3X13zQgx6U+/fvH8pz3XXXXZdm5qlDebIRanT8HHHEEbzoRS+qexm12rVrV91LkCSNwe7duwHYsWPHd8b5dffv388JJ5wwlOfasWPHMUN5ohFz26vhpqen616CJEmdYvy0gAEkSdLwGD8tYQBJUnf1trw0HsZPixhAkiQNzvhpGQNIkrrFqc/4GT8tZABJkrRyxk9LGUCSJK2M8dNiBpAktZtbXvUwflrOAJIkaXmMnw6Ynp42giSpZZz61Mf46RADSJKkxRk/HWMASVLzOfWpl/HTQQaQJEnzM346ygCSJGluxk+HGUCS1DxuedXP+Ok4A0iSpAMZPxPAAJKkZnDq0wzGz4QwgCRJKhg/E8QAkiTVKSJOjYhrI2JnRJwzx/3rIuIzEfG1iPh6RLygct8by8ddGxHPW+pzzsX4mTAGkCTVY9K3vCLiEOAC4PnAicDGiDix77Q/AC7OzCcCG4B3l489sbz9OOBU4N0RccgSn/Mgxs8EMoAkSTU4BdiZmbOZeTewGTit75wEHlx+/hDgxvLz04DNmbk3M3cBO8vnW8pzHmRq4L+KWqkXQLt27ap5JZLUfU2e+qxatYp169YN5bl27NhxTERsqxzalJmbys/XADdU7tsDPLnvKc4F/ikifhs4HPh3lcd+se+xa8rPF3vOgyw6+YmICyPi5oj45hz3vT4iMiKOKW9HRPxFue/29Yh4UuXc0yPiW+XH6Yt9XY2HUyBJ0hDdmpknVz42Ve6LOc7PvtsbgQ9k5lrgBcDfRMT9FnjsUp7zIEvZ9voAxf7aASLieOA5QDVnnw+sLz/OBN5TnnsU8GaKGjsFeHNErF7C19YYGECSNDpNnvqM2R7g+Mrttdy3rdVzBnAxQGZeDhwGHLPAY5fynAdZNH4y83PA7XPcdT7wuxxYWKcBH8rCF4EjI+JhwPOAyzLz9sy8A7iMOYJK9TGAJEkjdgWwPiKmI2IVxQXMW/rO2Q08GyAiHksRP7eU522IiEMjYppiyPLlJT7nQVZ0wXNEvBD4bmZe1XfXXPt5axY4PtdznxkR2yJi21133bWS5WmFDCBJ0qhk5j7gLOBS4BqKV3Vtj4jzyq4A+B3gVRFxFXAR8MpyoLKdYiJ0NfBJ4NWZuX++51xsLcu+4DkiHgj8PvDcue6e49iy9uTK/cFNAMcee+yi+3YarunpaS+ClqQhccvrQJm5Fdjad+xNlc+vBp42z2PfBrxtKc+5mJVMfn4GmAauiohvU+yvfTUi/g1D3pNTPZwASZK6bNnxk5nfyMyHZuYJmXkCRdg8KTP/F8U+2yvKV309Bfh+Zt5EMY56bkSsLi90fm55TA1lAEnSYJz6NNdSXup+EXA58OiI2BMRZyxw+lZgluKXD/0V8FsAmXk78BaKC5OuAM4rj6nBDCBJUhctes1PZm5c5P4TKp8n8Op5zrsQuHCZ61PNvAZIkpbPqU+z+fYWWpQTIElSlxg/WpLp6WkjSJLUCcaPlsUAkqSFueXVfMaPls0AkiS1mfGjFTGAJOlgTn3awfjRihlAkqQ2Mn40EANIktQ2xo8GZgBJkltebWL8aCgMIElSWxg/GhoDSNKkcurTLsaPhsoAkiQ1nfGjoTOAJE0Spz7tY/xoJAwgSVJTLfqu7tJK9QLId4WXpOZatWrVxP0Hq5Mfjdyk/aOSNDnc8mon40djYQBJkprC+NHYGECSusSpT3sZPxorA0iSVDfjR2NnAEmS6mT8qBYGkKQ2c8ur3Ywf1cYAkiTVwfhRrQwgSW3j1Kf9jB/VzgCSJI2T8aNGMIAktYFTn24wftQYBpAkaRyMHzXK9PS0ESRJGinjR41kAElqGre8usP4UWMZQJKkUTB+1GgGkKQmcOrTLcaPGs8AkiQNk/GjVjCAJEnDYvyoNQwgSXVwy2t4IuLUiLg2InZGxDlz3H9+RFxZflwXEXdW7vuTiNgeEddExF9ERPQ9dktEfHMp6zB+1CoGkCS1U0QcAlwAPB84EdgYESdWz8nM12bmSZl5EvAu4KPlY58KPA14AvB44BeAZ1Se+z8AP1zqWowftY4BJGlcnPoM1SnAzsyczcy7gc3AaQucvxG4qPw8gcOAVcChwP2B7wFExIOA1wFvXepCjB+1kgEkSY10TERsq3ycWblvDXBD5fae8thBIuIRwDTwaYDMvBz4DHBT+XFpZl5Tnv4W4L8BP17qIqeWeqLUNNPT0+zatavuZUjqqEmZ+hx66KHMzMwM6+luzcyT57kv5jiW85y7AbgkM/cDRMQjgccCa8v7L4uIpwP/CjwyM18bEScsdZFOftRqToAkqTX2AMdXbq8Fbpzn3A3ct+UF8H8AX8zMH2bmD4F/BJ4C/CLw8xHxbeBfgEdFxGcXW4jxo9bz/cAkqRWuANZHxHRErKIInC39J0XEo4HVwOWVw7uBZ0TEVETcn+Ji52sy8z2Z+fDMPAH4t8B1mfnMxRZi/KgzDCBJwzIpW17jlJn7gLOAS4FrgIszc3tEnBcRL6ycuhHYnJnVLbFLgOuBbwBXAVdl5t+vdC1e86NO8TogSWquzNwKbO079qa+2+fO8bj9wH9a5Lm/TfEy+EU5+VHnOAGSNAinPt1n/KiTDCBJ0nyMH3WWASRJmovxo04zgCQth1tek8H4UecZQJKkKuNHE8EAkrQYpz6TY9H4iYgLI+Lm6tvER8SfRsSOiPh6RHwsIo6s3PfG8q3qr42I51WOL/g29tKoGUCSJFja5OcDwKl9xy4DHp+ZTwCuA94IUL41/QbgceVj3h0RhyzlbeylcTCAJM3Fqc9kWTR+MvNzwO19x/6p/E2NAF/kvjcaO43itzLuzcxdwE6Kt7Bf7tvYSyNjAEnSZBvGNT//keINxmD+t6tfztvYnxkR2yJi21133TWE5UkH8/3AJGlyDRQ/EfH7wD7gw71Dc5yWCxw/+GDmpsw8OTNPPuywwwZZnrQoA0iSW16TZ8XxExGnA78C/Hrlzcfme7v65byNvTRWBpAkTZYVxU9EnAqcDbwwM39cuWsLsCEiDo2IaWA98GWW+Db2Ul0MIGkyOfWZTEt5qftFwOXAoyNiT0ScAfwlcARwWURcGRHvBcjM7cDFwNXAJ4FXZ+b++d7GfiR/I2mFDCBJmgxTi52QmRvnOPz+Bc5/G/C2OY4f9Db2UtNMT0+za9euupchSRohf8Oz1McJkDQZ3PKaXMaPNAcDSJK6a9FtL2lSuQUmdZdTn/usWrWKdevW1b2MsXLyIy3ACZAkdY/xIy3CAJK6xamPjB9pCQwgSeoO40daIt8PTJK6wfiRlskAktrLLS+B8SOtiAEkSe1l/EgrZABJ7eLURz3GjzQAA0iS2sf4kQZkAElSuxg/0hAYQFKzueWlKuNHGhIDSJLawfiRhsgAkprHqY/6GT/SkBlAktRsxo80AgaQ1AxOfTQX40caEQNIkprJ+JFGyPcDk6TmMX6kMTCApPFzy0vzMX6kMTGAJKkZjB9pjAwgaTyc+jRTRJwaEddGxM6IOGeO+8+PiCvLj+si4s7Kfesi4p8i4pqIuDoiTiiPPzsivlo+5l8i4pGLrcP4kcbMAJI0iSLiEOAC4PnAicDGiDixek5mvjYzT8rMk4B3AR+t3P0h4E8z87HAKcDN5fH3AL9ePuZ/An+w2FqMH6kGBpCkCXQKsDMzZzPzbmAzcNoC528ELgIoI2kqMy8DyMwfZuaPy/MSeHD5+UOAGxdbSKPj59BDD617CdLIGEDSaLjlVatjImJb5ePMyn1rgBsqt/eUxw4SEY8ApoFPl4ceBdwZER+NiK9FxJ+WkySA3wS2RsQe4OXA2xdb5NTy/k7jNzMzw+zsbN3LkEZienqaXbt21b0MSRNs1apVrFkzZ4OsxK2ZefI898Ucx3KeczcAl2Tm/vL2FPBLwBOB3cBHgFcC7wdeC7wgM78UEW8A/pwiiObV6MlPz8zMDDMzM3UvQxoJJ0DS8Dj1abQ9wPGV22uZf4tqA+WWV+WxXyu3zPYBHweeFBHHAj+XmV8qz/sI8NTFFtKK+OkxgNRVBpCkCXAFsD4ipiNiFUXgbOk/KSIeDawGLu977OoydgCeBVwN3AE8JCIeVR5/DnDNYgtpVfyAAaTuMoCkwTj1abZyYnMWcClFoFycmdsj4ryIeGHl1I3A5szMymP3A68HPhUR36DYQvur8jlfBfxdRFxFcc3PGxZbS+Ov+ZlLL4C8Fkhd4zVAkrosM7cCW/uOvanv9rnzPPYy4AlzHP8Y8LHlrKN1k58qp0DqIt8PTJJGq9XxAwaQussAkpbOLS8tR+vjB3w1mLrLAJKk4etE/PQYQOoiA0hamFMfLVen4gcMIHWTASRJw9O5+AG3wdRNBpAkDUcn46fHAFLXGEDSgdzy0kp0On7AAFL3GECSNJjOxw+4DabuMYAkpz5auYmInx4DSF1iAEnSykxU/IABpG4xgDSpnPpoEBMXP+A2mLrFt8OQpOWZyPjpMYDUJQaQJC3NRMcPGEDqFgNIk8AtLw1q4uMH3AZTtxhAkrSwReMnIi6MiJsj4puVY0dFxGUR8a3yz9Xl8YiIv4iInRHx9Yh4UuUxp5fnfysiTh/NX2cwBpC6wgBSVzn10TAsZfLzAeDUvmPnAJ/KzPXAp8rbAM8H1pcfZwLvgSKWgDcDTwZOAd7cC6amMYDUFQaQJM1tarETMvNzEXFC3+HTgGeWn38Q+Cxwdnn8Q5mZwBcj4siIeFh57mWZeTtARFxGEVQXDfw3GIFeAM3Ozta8Emkw09PT7Nq1q+5lSEPh1Gc0pqamOO644+pexlit9Jqf4zLzJoDyz4eWx9cAN1TO21Mem+/4QSLizIjYFhHbfvSjH61wecPhFEhd4ARIkg407AueY45jucDxgw9mbsrMkzPz5MMPP3yoi1sJA0hdYABJ0n1WGj/fK7ezKP+8uTy+Bzi+ct5a4MYFjreCASRJUnesNH62AL1XbJ0OfKJy/BXlq76eAny/3Ba7FHhuRKwuL3R+bnmsNQwgtZ3TH0kqLOWl7hcBlwOPjog9EXEG8HbgORHxLeA55W2ArcAssBP4K+C3AMoLnd8CXFF+nNe7+LlNDCBJqs+6devqXoI6Yimv9to4z13PnuPcBF49z/NcCFy4rNU10MzMjK8CU2v56i9J8jc8r4gTIEmS2sv4WSEDSG3ltT+SJp3xMwADSG1lAEmaZMbPgAwgSZLaxfgZAt8VXm3k9EfSpDJ+hsgAkiSp+YyfITOA1CZOfyRNIuNnBAwgSZKay/gZEQNIbeH0R9KkMX5GyACSJKl5jJ8RM4DUBk5/JE0S42cMDCBJkprD+BkTA0hN5/RH0qhFxKkRcW1E7IyIc+a4//yIuLL8uC4i7qzct79y35bK8Q+Xz/nNiLgwIu6/2DqMnzEygNR0BpCkUYmIQ4ALgOcDJwIbI+LE6jmZ+drMPCkzTwLeBXy0cvdPevdl5gsrxz8MPAb4WeABwG8uthbjZ8wMIEnShDoF2JmZs5l5N7AZOG2B8zcCFy32pJm5NUvAl4G1iz1maokL1hDNzMwwOztb9zKkOU1PT7Nr1666lyFpTKampjj66KOH9XTHRMS2yu1Nmbmp/HwNcEPlvj3Ak+d6koh4BDANfLpy+LDyufcBb8/Mj/c95v7Ay4HXLLZI46cmBpAkqYNuzcyT57kv5jiW85y7AbgkM/dXjq3LzBsjYgb4dER8IzOvr9z/buBzmfn5xRbptleN3AJTU3ntj6QR2AMcX7m9FrhxnnM30LfllZk3ln/OAp8Fnti7LyLeDBwLvG4pCzF+amYASZImxBXA+oiYjohVFIGzpf+kiHg0sBq4vHJsdUQcWn5+DPA04Ory9m8CzwM2ZuZPl7IQ46cBDCA1kdMfScOUmfuAs4BLgWuAizNze0ScFxHVV29tBDaXFzD3PBbYFhFXAZ+huObn6vK+9wLHAZeXL4N/02Jr8ZqfhugFkNcBSZK6KjO3Alv7jr2p7/a5czzu/6d4Kftcz7nslnHy0zBOgdQkTn8kdZHx00AGkJrEAJLUNcZPQxlAkiSNhvHTYAaQmsLpj6QuMX4azgCSJGm4jJ8WMIDUBE5/JHWF8dMSBpAkScNh/LSIAaS6Of1R3datW1f3EtQBxk/LGECSJA3G+GkhA0h1cvojqe2Mn5YygCRJWhnjp8UMINXF6Y+kNjN+Ws4AUl0MIEltZfx0gAEkSdLSGT8dYQCpDk5/JLWR8dMhMzMzRpAkSYswfjrIANI4Of2R1DZTdS9AozEzM8Ps7Gzdy5AkNdzU1BRHH3VU3csYKyc/HeYESOPi9EdSmxg/HWcASZJ0IONnAhhAGgenP5LawviZEAaQJEkF42eCGEAaNac/ktrA+JkwBpBGzQCS1HQDxU9EvDYitkfENyPioog4LCKmI+JLEfGtiPhIRKwqzz20vL2zvP+EYfwFtHwGkCRpkq04fiJiDfB/Aydn5uOBQ4ANwB8D52fmeuAO4IzyIWcAd2TmI4Hzy/NUEwNIo+T0R1KTDbrtNQU8ICKmgAcCNwHPAi4p7/8g8Kvl56eVtynvf3ZExIBfXwMwgCRJk2jF8ZOZ3wX+DNhNET3fB74C3JmZ+8rT9gBrys/XADeUj91Xnn/0Sr++hsMA0qg4/ZHUVINse62mmOZMAw8HDgeeP8ep2XvIAvdVn/fMiNgWEdt+9KMfrXR5WgYDSJI0SQbZ9vp3wK7MvCUz7wE+CjwVOLLcBgNYC9xYfr4HOB6gvP8hwO39T5qZmzLz5Mw8+fDDDx9geVoOA0ij4PRHUhMNEj+7gadExAPLa3eeDVwNfAZ4cXnO6cAnys+3lLcp7/90Zh40+VF9DCBJ0iQY5JqfL1FcuPxV4Bvlc20CzgZeFxE7Ka7peX/5kPcDR5fHXwecM8C6NSIzMzNGkIbK6Y+kppla/JT5ZeabgTf3HZ4FTpnj3LuAlwzy9TQ+MzMzzM7O1r0MSZKGzt/wrHk5AdKwOP2R1CTGjxZkAGlYDCBJTWH8aFEGkCSpS4wfLYkBpGFw+iNNtog4NSKuLd/n86AXPkXE+RFxZflxXUTcWR5/RER8pTy+PSL+r8pjVkXEpvL8HRHxosXWMdAFz5osXgQtqQnWrVvH7t27616GlikiDgEuAJ5D8bv/roiILZl5de+czHxt5fzfBp5Y3rwJeGpm7o2IBwHfLB97I/D7wM2Z+aiIuB9w1GJrcfKjZXECpEE5/ZEm1inAzsyczcy7gc0U7xQxn43ARQCZeXdm7i2PH8qB/fIfgf9anvfTzLx1sYU4+dGyOQGSpA7Zt5e8Zdewnu2YiNhWub0pMzeVn9/7Hp+lPcCT53qSiHgExdtnfbpy7HjgH4BHAm/IzBsj4sjy7rdExDOB64GzMvN7Cy3SyY9WxAmQBuH0R+qsW3tvUVV+bKrct6T3+CxtAC7JzP33nph5Q2Y+gSJ+To+I4yiGOGuBL2Tmk4DLKd50fUHGj1bMAJIkLcO97/FZqr7/Z78NlFte/crrfLYDvwTcBvwY+Fh5998CT1psIcaPBmIAaaWc/kgT5wpgfURMR8QqisDZ0n9SRDwaWE0xxekdWxsRDyg/Xw08Dbi2fI/QvweeWZ7ae5/RBRk/GpgBpJUygKTJkZn7gLOAS4FrgIszc3tEnBcRL6ycuhHY3Pfm548FvhQRVwH/DPxZZn6jvO9s4NyI+DrwcuB3FluLFzxrKLwIWpK0mMzcCmztO/amvtvnzvG4y4AnzPOc3wGevpx1OPnR0DgB0ko4/ZE0bsaPhsoAkiQ1nfGjoZuZmTGCtCxOfySNk/GjkTGAJElNZPxopAwgLZXTH0nj0uj4WbVqVd1L0BAYQJKkJml0/EDx7r1qPwNIS+H0R9I4ND5+wADqCgNIktQErYgfMIC6wgDSYpz+SBq11sQPGEBdYQBpMQaQpFFqVfxAEUBGUPsZQJKkurQufnoMoPYzgLQQpz+SRqW18QMGUBcYQJKkcWt1/IAB1AUGkObj9EfSKLQ+fsAA6gIDSJI0Lp2IHzCAusAA0lyc/kgats7ED/hKsC4wgCRJozZV9wJGYd26dezevbvuZWiFZmZmmJ2drXsZapDp6Wl27dpV9zKkTsp79rL/pp11L2OsOjX5qXIC1G4zMzNOgSRJI9HZ+AEDqAsMIPV47Y+kYel0/IAB1AUGkHoMIEnD0Pn4AQOoCwwgSdKwTET8gK8E6wIDSOD0RwV/nmsQExM/Pf6DaTcDSJI0qImLHzCA2s4AktMfSYOYyPgBA6jtDCBJ0kpNbPyAAdR2BtBkc/ojaaUmOn7AAGo7A0iStFwTHz9gALWdATS5nP5IWgnjp+RL4dvNAJpcBpCk5TJ++hhA7WUASZKWwviZgwHUXgbQZHL6I2k5jJ95GEDtZQBJkhZi/CzAAGovA2jyOP2RtFQDxU9EHBkRl0TEjoi4JiJ+MSKOiojLIuJb5Z+ry3MjIv4iInZGxNcj4knD+SuMlgHUXjMzM0aQJOkgg05+3gl8MjMfA/wccA1wDvCpzFwPfKq8DfB8YH35cSbwngG/9tj4SrB2M4Amh9Mfqdki4tSIuLYchJwzx/3nR8SV5cd1EXFnefykiLg8IraXA5RfqzxmOiK+VA5dPhIRqxZbx4rjJyIeDDwdeD9AZt6dmXcCpwEfLE/7IPCr5eenAR/KwheBIyPiYSv9+nUwgNrLAJKkekXEIcAFFMOQE4GNEXFi9ZzMfG1mnpSZJwHvAj5a3vVj4BWZ+TjgVOAdEXFked8fA+eXQ5c7gDMWW8sgk58Z4Bbgv0fE1yLiryPicOC4zLyp/EvcBDy0PH8NcEPl8XvKYweIiDMjYltEbPvBD34wwPJGwwBqLwNoMjj9kRrrFGBnZs5m5t3AZorByHw2AhcBZOZ1mfmt8vMbgZuBYyMigGcBl5SPqQ5d5jVI/EwBTwLek5lPBH7EfVtcc4k5juVBBzI3ZebJmXnyEUccMcDyRscAai8DSJJG6pjeAKP8OLNy35KGIAAR8QhgGvj0HPedAqwCrgeOBu7MzH2LPWfV1FL+JvPYA+zJzC+Vty+hiJ/vRcTDMvOmclvr5sr5x1cevxa4cYCvX6t169axe/fuupehFZiZmWF2drbuZWiEpqen2bVrV93LkFoh997FPd/ZMaynuzUzT57nviUNQUobgEsyc/8BT1B0xd8Ap2fmT8vJz1Kf814rnvxk5v8CboiIR5eHng1cDWwBTi+PnQ58ovx8C/CK8lVfTwG+39seaysnQO3lBKj73P6SGmc5Q5ANlFtePeW1xv8A/EF57TDArRTXEPeGOUsarAwy+QH4beDD5ZXVs8BvUATVxRFxBrAbeEl57lbgBcBOiguXfmPAr90IvQByCtQ+ToAkaayuANZHxDTwXYrAeVn/SeVQZTVweeXYKuBjFC+c+tve8czMiPgM8GKKa4iqQ5d5DRQ/mXklMNd469lznJvAqwf5ek3mNlg7GUDd5vaX1ByZuS8izgIuBQ4BLszM7RFxHrAtM7eUp24ENpfd0PNSileYHx0RryyPvbLskLOBzRHxVuBrlK9CX8igkx9VGEDtZABJ0nhk5laKnaDqsTf13T53jsf9D+B/zPOcsxSvJFsy395iyLwOqJ28Bqi7vPZHUj/jZwQMoHYygCRpMhg/I2IAtZMB1E1OfyRVGT8j5HuCtZMBJEndZvyMgQHUPgZQ9zj9kdRj/IyJAdQ+BpAkdZPxM0YGUPvMzMwYQR3i9EcSGD9jZwC1kwHUHQaQJOOnBgZQOxlAUrP4s1QrZfzUxH+07WQAdYPTH2myGT818qXw7WQASVK7GT8NYAC1jwHUfk5/pMll/DSEAdQ+BpAktZPx0yAGUPsYQO3m9EeaTMZPwxhA7WMASVK7GD8NZAC1jwHUXk5/pMlj/DSUrwRrHwNIktphqu4FaGHr1q1j9+7ddS9DSzQzM8Ps7Gzdy9AyTU9Ps2vXrrqXIdXip3v38oPrrq97GWPl5KcFnAC1ixOgdnL7S5ocxk9LGEDtYgBJUnMZPy1iALWLAdQ+Tn+kyeA1Py1TDSCvBWq+agB5LVA79ALIa4DawZ+JWgnjp8X6J0H+w282Q6hdqlMgQ6gdej8T/VmoxRg/HeJ/AbWHIdQuhlC7+LNQi2l0/KxatYo1a9YA8N3vfrfm1bSL//jboxdCRlA7uC3WLv4s1FwaHT9VvQgCQ2i5/MffDk6D2sVpUPu4Laae1sRPVTWEwBhaDq8TagdDqF0MoXbxPwjVyvjpZwytnD8Ems9tsXYxhNrFn4GTqRPx088tspXxh0CzOQ1qH68Pahd/Bk6OTsZPlVOhlfGHQLMZQu3iNKh9vD6o2zofP/2MoeXzOqFmM4TaxRBqF/9DsJsmLn76GUPL5w+D5vL6oHYxhNrFn33dMfHx08/rhZbHHwbN5DSofbw+qF382ddujY6fqakpjjvuOAC+973vjf3rOxVaHn8YNJMh1C5Og9rH64Pap9HxU9WLoB5jqNkMoWZyW6xdDKF28edee7QmfvpVY6iOEAK3yJbKC6abx2lQ+7gt1i6GULO1Nn6qnAq1iz8UmsUQahenQe3jtth9IuJU4J3AIcBfZ+bb++4/H/jl8uYDgYdm5pHlfacDf1De99bM/GDfY7cAM5n5+MXW0Yn46WcMtYch1CyGULsYQu0y6T/vIuIQ4ALgOcAe4IqI2Cg9+ZgAABYpSURBVJKZV/fOyczXVs7/beCJ5edHAW8GTgYS+Er52DvK+/8D8MOlrqWT8dPPLbJ2mPQfDE3j9UHtYgi1y4T+vDsF2JmZswARsRk4Dbh6nvM3UgQPwPOAyzLz9vKxlwGnAhdFxIOA1wFnAhcvZSETET9VToXaweuEmsNpUPt4fVC7dCyEjomIbZXbmzJzU/n5GuCGyn17gCfP9SQR8QhgGvj0Ao/t/Z/pW4D/Bvx4qYucuPjpZwy1Q8d+OLSWIdQuToPaZ926dezYsWOsX3PfXXu5fcd3hvV0t2bmyfPcF3Mcy3nO3QBckpn7F3psRJwEPDIzXxsRJyx1kRMfP/2aFkOG0MEMoWYwhNrFEFID7AGOr9xeC9w4z7kbgFf3PfaZfY/9LPCLwM9HxLcpmuahEfHZzKyee5BGx8/U1BRHH300ALfddlsta6j7eiGnQgszhJrB64PaxRBSTa4A1kfENPBdisB5Wf9JEfFoYDVweeXwpcAfRcTq8vZzgTeW1wC9p3zcCcD/u1j4QMPjp6oXQT11xFDTpkJgDFV5nVD9nAa1j9cHaVwyc19EnEURMocAF2bm9og4D9iWmVvKUzcCmzMzK4+9PSLeQhFQAOf1Ln5eiYHjp3zp2jbgu5n5K2XRbQaOAr4KvDwz746IQ4EPAT8P3Ab8WmZ+e6Vf1xgquEU2P6dC9TKE2sVpkMYhM7cCW/uOvanv9rnzPPZC4MIFnvvbwKK/4weGM/l5DXAN8ODy9h8D52fm5oh4L3AGxUjqDOCOzHxkRGwoz/u1IXx94MAYcousYAzdxxCql9ti7WIIqesGip+IWAv8e+BtwOsiIoBncd8e3geBcyni57Tyc4BLgL+MiKiOtYbFqVDBGJqbIVQfp0Ht47aYumjQyc87gN8FjihvHw3cmZn7ytvV1+Hf+xr9ct/v++X5t1afMCLOpPhFRaxdu3bA5ZWLMoYAY2guXidUH0OoXZwGqUtWHD8R8SvAzZn5lYh4Zu/wHKfmEu6770Dxy5A2AZx00klDnwqBW2Q9Xi90MKdC9TCE2sUQUtsNMvl5GvDCiHgBcBjFNT/vAI6MiKly+lN9DX/v9f17ImIKeAiw4iu1h8WpUMGp0MEMoXp4fVC7GEJqoxXHT2a+EXgjQDn5eX1m/npE/C3wYopXfJ0OfKJ8yJby9uXl/Z8exfU+gzKGCsbQgQyh8XMa1D5eH6S2GMXv+Tkb2BwRbwW+Bry/PP5+4G8iYifFxGfDooubmuLoo4669/Ztt49/UNS0GHKLrH5eJzR+hlC7OA1S0w0lfjLzsxS/Zpry3VpPmeOcu4CXDPJ16g4hqP96IadCzeNUaLwMoXYxhNRErfkNz/2qIQROhXqMoXoZQuPl9UHt4raYmqK18dPPGCq4RdYchtD4OA1qF6dBqltn4qefW2ROhZrE64TGxxBqF0NIdehs/FQ5FSoYQ83hVGg83BZrF0NI4zIR8dPPGCo0LYYMIUNoVJwGtY/XB2mUJjJ++rlFVqj7eiGnQobQOBhC7eI0SKNg/PRxKlRo2lQIJi+GvE5o9AyhdjGENCzNjp99e8lb7vsfeBw7vcDJo2EMFZoWQ5MWQuBUaNS8PqhdDCENotnx06caQmAMgVtkPZMWQ4bQ6DgNah+vD9JytSp++tU9FYL6rxdyKlSY5BgyhEbHEGoXp0Ers+8n+7hl+y11L2OsWh0/VU6FyjUYQ8DkbpF5ndDouC3WLoaQFtKZ+OlnDJVrcIvMqVDJEBoOp0Ht47aY+nU2fvq5ReZUqGdSY8gQGj5DqF2cBqlnYuKnyqlQuQZjCJjMGDKEhs8QahdDaLJNZPz0M4bKNTQshnxj1vHwOqHh8/qgdjGEJk+j4yfv2VvP13WLrFiDb8zqVMgQGojToPbx+qDJ0Oj4Adh/0857Pz/kYY8c+9d3KlSuoWFTITCGxsEQGh5DqF2cBnVb4+OnqhpCYAz1GENukY2DITQ8hlC7GELd06r46WcMFdwicyo0bobQ8Hh9ULsYQt3Q6vjpV/cWGdR/vZBToYIxND5eMD0cToPax+uD2qtT8VPlVKhgDBWaFkNdDSFwKjQMhlC7OA1qn87GTz9jqOAWWaHu64UmcSpkCK2M22LtYgi1w8TETz+3yJwK9TRtKgTdjCFDaDBOg9rHbbHmmtj4qXIqVDCGCk2Loa6HEBhDy2UItYvToOZpdPzk3rtq+brGUKFpMeQWWaHrMWQILY8h1C6GUDM0On4A7vnOjgNu3/8Rjxn7GpoWQ5P6W6edChW6HkOG0Mp5fVC7GEL1aXz89KvGUB0hBPVfL+RUqFyDMQR0e4vMEFoZp0Ht4/VB49W6+KlyKlQwhso1uEXW6amQ1wmtjCHULtVpkEan1fHTzxgquEXmVKhnUmLIEFoaQ0h1i4hTgXcChwB/nZlvn+OclwLnAglclZkvK4//CfDvgfsBlwGvycyMiI3A75Xn3wj8n5l560Lr6FT89HOLzKnQvWswhoDuxpAhtHxeH6Rxi4hDgAuA5wB7gCsiYktmXl05Zz3wRuBpmXlHRDy0PP5U4GnAE8pT/wV4RkT8C0VMnZiZt5aBdBZFPM2r0/FT5VSoYAyVa2hYDPnGrMNjCC2P0yCN0SnAzsycBYiIzcBpwNWVc14FXJCZdwBk5s3l8QQOA1YBAdwf+F75eQCHR8RtwIOBA//Pdg4TEz/9jKGCW2TlGnxj1k5OhbxOaHkMIQ3BMRGxrXJ7U2ZuKj9fA9xQuW8P8OS+xz8KICK+QLE1dm5mfjIzL4+IzwA3UcTOX2bmNeW5/xn4BvAj4FvAqxdb5MTGTz+3yJwK3buGhk2FwBgaFqdCS+e22OS4Z+9+brz+jmE93a2ZefI898Ucx7Lv9hSwHngmsBb4fEQ8HjgGeGx5DOCyiHg6cDnwn4EnArPAuyi2zd660CIbHT8/3bu3lq/rVKhgDJVraFgMuUU2HIbQ0jgN0hDtAY6v3F5LcYFy/zlfzMx7gF0RcS33xdAXM/OHABHxj8BTgJ8AZOb15fGLgXMWW0ij4wfgB9ddf8DtIx71M2NfgzFUaFoMuUVWcCo0OENoaQwhDegKYH1ETAPfBTYAL+s75+PARuADEXEMxTbYLDADvCoi/ivFBOkZwDvK5zkxIo7NzFsoLqa+ZrGFND5++hlDhbq3yKD+64WcChWMoeHyOqGlMYS0XJm5LyLOAi6luJ7nwszcHhHnAdsyc0t533Mj4mpgP/CGzLwtIi4BnkVxbU8Cn8zMvweIiD8EPhcR9wDfAV652Fois3+7rTlOOv64/KfXbFjy+XWEUL+6rheqqiuGquq6eLqqrsnQAWuo6Zct9tS1RVbV5hDqZwgtzAgajve9731fWeC6maH7mfs/IP9o9XB+Zm+45Zqxrn2lWjf5WYhToYJbZAW3yJwKDZvbYwtzGqS26FT89DOGCm6RuUXWYwwNjyG0MENITdbp+OlXjaG6tsjqfkm9U6GCMVRoWgx1IYTAGOrny+bVNBMVP1VOhQrGUKFpMeQbsxa6EEOG0H2cBqkpJjZ++hlDBbfICnVfL+RUqNCFGDKE5mYIqU6Njp99d+3l9h3f4ajHPGLsX9stMqdCPU2bCoExBIZQlxhCGrdGx0/P7Tu+c+/ndYcQOBXqMYYKdceQW2SFtsWQ1wnNzeuDNA6tiJ+qagiBMdRjDLlFdu8anAoB7Y4hQ8hpkEardfHTzxgq1L1FBvVfL+RUqFyDMQS0O4YMoQMZQhq2FcdPRBwPfAj4N8BPKd62/p0RcRTwEeAE4NvASzPzjogI4J3AC4AfA6/MzK8OtvyD1b1FBvVfL+RUqGAMlWtwiwxo7/VChtCBDCENwyCTn33A72TmVyPiCOArEXEZxXtqfCoz3x4R51C8u+rZwPMp3pl1PfBk4D3lnyPjVKhgDBXcInMq1NPWqZDXCR3I64O0UiuOn8y8Cbip/PwHEXENsAY4jeKt5wE+CHyWIn5OAz6UxZuJfTEijoyIh5XPMxbGUMEtMqdC967BGAK6EUOTHEJOg7RcQ7nmJyJOAJ4IfAk4rhc0mXlTRDy0PG0NcEPlYXvKYwfET0ScCZwJ8PDDHzCM5c3LLTKnQj3GULmGhsWQW2RLZwgVDCEtxcDxExEPAv4O+C+Z+a/FpT1znzrHsYPeUj4zNwGbAB5/7JFje8t5p0IFY6jQtBjyjVkLToWWxhAquC2m+QwUPxFxf4rw+XBmfrQ8/L3edlZEPAy4uTy+Bzi+8vC1wI0LPf++n+zjlu23cOzjjh1kmStiDBXcIivUfb2QU6GCMbR8hpDTIB1skFd7BfB+4JrM/PPKXVuA04G3l39+onL8rIjYTHGh8/eXer3PLdtvOeB23THkFlnBqVDBGCrX4BZZq0IIJjOGDCHBYJOfpwEvB74REVeWx36PInoujogzgN3AS8r7tlK8zH0nxUvdf2OlX7gaQ3WHEDgV6jGGCnXHkFtkBadCi5v0qZAhNLmiePFVMz3mQQ/Kv/rZn1vWY+qIoX51TYaq6no/sqq6tsl66toiq6rrJfVVdcXQAWuo6fcLVdU1Gapqegz1TGII9TQhgt73vvd9JTNPHtfXe/j9Ds0zVq0dynO9de/sWNe+Uq3/Dc/9mrZFBvVPhnxj1oJToYJbZIW6J0NNDqFJngg5DZoMnYuffnVvkUH91wu5RVYwhgpukRXqvl6oLVtkk3ydkCHUXZ2PnyqnQgVjqOCryJwK9TRtKgTtiCFDSG01UfHTzxgquEXmVKjHGCo0LYYMoWbx9we130THTz+3yJwK9RhDhabFkFtkhSbG0CSGkNOg9mp0/Nyzdz83Xn8HD/+Z1WP/2k6FCsZQoWkx5BuzlmtwKgQ0L4Ym8TohQ6hdGh0/PTdef8cBt42hQt0xNKlbZFD/9UJOhco1GENAu2JokkLICGquVsRPv6bFkFtkBadCBWOoUHcMTeoWGTT7eqFJCiGnQc3VyvjpV42hukMInAr1GEOFumPILbJyDU6FgGbFkCGkunQifqqaNhUCY6in7hhyi6zgVKhcgzEENCeGJuk6IUOofp2Ln37GUKHuLTKo/3ohp0IFY6hcQ8NiyC2yA03KVMjrg+rR+fjpV/cWGdR/vZBToYIxVGhaDE3qb512KjS/SQih6jRIozdx8VPlVKhgDBXcIivUfb2QU6GCMTS3SQghjd5Ex08/Y6jgFplToZ6mTYXAGAK3yHom6TohDVej4+cn+3/KN/91LwCPf/ChY//6bpE5FeoxhgpNiyG3yApOhQpOhbRUjY6fql4EQf0hBE6FeoyhQt0x5BZZwalQwRgyhLSw1sRPVTWEwBjqMYYm97dOOxUqGEOFpsWQIaSmaWX89GtaDE3qFhnUf72QU6GCMVRwi6xQ9/VCTZoKeZ2QoCPx088tMqdCPcZQwS0yp0I9TZsKQXNiyBAavYg4FXgncAjw15n59jnOeSlwLpDAVZn5soj4ZeD8ymmPATZk5scj4sPAycA9wJeB/5SZ9yy4jswcxt9nJB5+v0PzjFVrh/qcdcRQv7omQ1V1TYaq6nolWU9dW2RVdb2kvqquGKqq6y04quqaDB2whpomQ1V1vZKsp+4tsp46Q+jss8/+SmaePK6vN8z/r33r3tl51x4RhwDXAc8B9gBXABsz8+rKOeuBi4FnZeYdEfHQzLy573mOAnYCazPzxxHxAuAfy7v/J/C5zHzPQuvs5ORnIU3bIoP6J0NukRWcChXcIivUPRlyi6xQVww5ERqJU4CdmTkLEBGbgdOAqyvnvAq4IDPvAOgPn9KLgX/MzB+X52zt3RERXwYWLblGx89Nefetb907+53FzxzALYufMgTHALfWvIb5fbHmrz9eC38vNE5+L5rF70dzPHqcX+ymvPvSt+6dPWZIT3dYRGyr3N6UmZvKz9cAN1Tu2wM8ue/xjwKIiC9QbI2dm5mf7DtnA/Dn/V84Iu4PvBx4zWKLbHT8ZGb9ezNDEBHbxjnC1Pz8XjSH34tm8fvRHH3xMHKZeeqYvlTM9eX7bk8B64FnUkxwPh8Rj8/MOwEi4mHAzwKXzvFc76bY8vr8Ygu53zIWLUmStFJ7gOMrt9cCN85xzicy857M3AVcSxFDPS8FPtZ/QXNEvBk4FnjdUhZi/EiSpHG4AlgfEdMRsYpi+2pL3zkfB34ZICKOodgGq77l/UbgouoDIuI3gedRXDz906UsxPgZj02Ln6Ix8XvRHH4vmsXvR3N08nuRmfuAsyi2rK4BLs7M7RFxXkS8sDztUuC2iLga+Azwhsy8DSAiTqCYHP1z31O/FzgOuDwiroyINy22lka/1F2SJGnYnPxIkqSJYvxIkqSJYvwMKCJOjYhrI2JnRJwzx/2HRsRHyvu/VO5ZEhHPiYivRMQ3yj+fNe61d81KvxeV+9dFxA8j4vXjWnNXDfK9iIgnRMTlEbG9/Pdx2DjX3jUD/Iy6f0R8sPweXBMRbxz32rtmCd+Lp0fEVyNiX0S8uO++0yPiW+XH6eNbdUdlph8r/KD4BUzXAzPAKuAq4MS+c34LeG/5+QbgI+XnTwQeXn7+eOC7df992vwxyPeicv/fAX8LvL7uv0+bPwb8dzEFfB34ufL20cAhdf+d2vox4PfiZcDm8vMHAt8GTqj779TWjyV+L04AngB8CHhx5fhRFK94OgpYXX6+uu6/U5s/nPwM5t5f1Z2ZdwO9X9VddRrwwfLzS4BnR0Rk5tcys/f7DbZT/FbM+t94rL1W/L0AiIhfpfiBsn1M6+2yQb4XzwW+nplXAWTmbZm5f0zr7qJBvhcJHB4RU8ADgLuBfx3Psjtp0e9FZn47M78O9L9c+3nAZZl5exZv+3AZMK5fTNhJxs9g5vpV3WvmOyeLl/l9n+K/ZqteBHwtM/eilVrx9yIiDgfOBv5wDOucBIP8u3gUkBFxaTn+/90xrLfLBvleXAL8CLgJ2A38WWbW/+6v7bWU78UoHqs5NPrtLVpgKb+qe8FzIuJxwB9T/BevVm6Q78UfAudn5g/LQZAGM8j3Ygr4t8AvAD8GPhURX8nMTw13iRNjkO/FKcB+4OEUWy2fj4j/L8s3pdSyLeV7MYrHag5Ofgaz1F/VfTxAOT5+CHB7eXst8DHgFZl5PRrEIN+LJwN/EhHfBv4L8HsRcdaoF9xhg3wv9gD/nJm3ZvGOzVuBJ418xd01yPfiZcAns3ibgZuBLwC+99fKLeV7MYrHag7Gz2CW8qu6twC9K/NfDHw6MzMijgT+AXhjZn5hbCvurhV/LzLzlzLzhMw8AXgH8EeZ+ZfjWngHrfh7QfHbXZ8QEQ8s/4/4GcDVY1p3Fw3yvdgNPCsKhwNPAXaMad1dtJTvxXwuBZ4bEasjYjXFTsFcb+ypJTJ+BpBL+1Xd76e4rmQnxRuu9V7eeBbwSOD/KX8d95UR8dAx/xU6Y8DvhYZokO9FeTHnn1P8H8WVwFcz8x/G/XfoigH/XVwAPAj4JsX347+XF+NqBZbyvYiIX4iIPcBLgPdFxPbysbcDb6H4PlwBnOf1V4Px7S0kSdJEcfIjSZImivEjSZImivEjSZImivEjSZImivEjSZImivEjSZImivEjSZImyv8Ghd5DlcWjKDwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.contourf(LR, NE, Z, 20, cmap='RdGy')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "dc = pd.read_csv(\"Hyperparam_search/Graident_Tree_Boosting_CVAE_LS_GS_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_n_iter_no_change</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "      <th>...</th>\n",
       "      <th>std_test_accuracy</th>\n",
       "      <th>rank_test_accuracy</th>\n",
       "      <th>split0_test_F1</th>\n",
       "      <th>split1_test_F1</th>\n",
       "      <th>split2_test_F1</th>\n",
       "      <th>split3_test_F1</th>\n",
       "      <th>split4_test_F1</th>\n",
       "      <th>mean_test_F1</th>\n",
       "      <th>std_test_F1</th>\n",
       "      <th>rank_test_F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>135.228478</td>\n",
       "      <td>40.145879</td>\n",
       "      <td>0.535533</td>\n",
       "      <td>0.153077</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1000, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002537</td>\n",
       "      <td>1</td>\n",
       "      <td>0.788781</td>\n",
       "      <td>0.803516</td>\n",
       "      <td>0.795395</td>\n",
       "      <td>0.802655</td>\n",
       "      <td>0.787875</td>\n",
       "      <td>0.795645</td>\n",
       "      <td>0.006613</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>63.831421</td>\n",
       "      <td>21.568672</td>\n",
       "      <td>0.267960</td>\n",
       "      <td>0.084825</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1500, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005036</td>\n",
       "      <td>2</td>\n",
       "      <td>0.785106</td>\n",
       "      <td>0.798503</td>\n",
       "      <td>0.796936</td>\n",
       "      <td>0.801955</td>\n",
       "      <td>0.781604</td>\n",
       "      <td>0.792821</td>\n",
       "      <td>0.007975</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>72.927708</td>\n",
       "      <td>36.371233</td>\n",
       "      <td>0.306629</td>\n",
       "      <td>0.144226</td>\n",
       "      <td>0.10</td>\n",
       "      <td>500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 500, 'n...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007360</td>\n",
       "      <td>3</td>\n",
       "      <td>0.789289</td>\n",
       "      <td>0.793782</td>\n",
       "      <td>0.803340</td>\n",
       "      <td>0.804890</td>\n",
       "      <td>0.772151</td>\n",
       "      <td>0.792690</td>\n",
       "      <td>0.011805</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>442.726007</td>\n",
       "      <td>16.072229</td>\n",
       "      <td>1.697251</td>\n",
       "      <td>0.007759</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1500, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003054</td>\n",
       "      <td>4</td>\n",
       "      <td>0.783275</td>\n",
       "      <td>0.797217</td>\n",
       "      <td>0.789154</td>\n",
       "      <td>0.801060</td>\n",
       "      <td>0.779098</td>\n",
       "      <td>0.789961</td>\n",
       "      <td>0.008237</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>284.701999</td>\n",
       "      <td>8.511558</td>\n",
       "      <td>1.137838</td>\n",
       "      <td>0.012376</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1000, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003312</td>\n",
       "      <td>5</td>\n",
       "      <td>0.774004</td>\n",
       "      <td>0.788307</td>\n",
       "      <td>0.779025</td>\n",
       "      <td>0.790700</td>\n",
       "      <td>0.773780</td>\n",
       "      <td>0.781164</td>\n",
       "      <td>0.007104</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "2           2     135.228478     40.145879         0.535533        0.153077   \n",
       "3           3      63.831421     21.568672         0.267960        0.084825   \n",
       "1           1      72.927708     36.371233         0.306629        0.144226   \n",
       "7           7     442.726007     16.072229         1.697251        0.007759   \n",
       "6           6     284.701999      8.511558         1.137838        0.012376   \n",
       "\n",
       "   param_learning_rate  param_n_estimators  param_n_iter_no_change  \\\n",
       "2                 0.10                1000                      20   \n",
       "3                 0.10                1500                      20   \n",
       "1                 0.10                 500                      20   \n",
       "7                 0.01                1500                      20   \n",
       "6                 0.01                1000                      20   \n",
       "\n",
       "   param_subsample                                             params  ...  \\\n",
       "2              0.7  {'learning_rate': 0.1, 'n_estimators': 1000, '...  ...   \n",
       "3              0.7  {'learning_rate': 0.1, 'n_estimators': 1500, '...  ...   \n",
       "1              0.7  {'learning_rate': 0.1, 'n_estimators': 500, 'n...  ...   \n",
       "7              0.7  {'learning_rate': 0.01, 'n_estimators': 1500, ...  ...   \n",
       "6              0.7  {'learning_rate': 0.01, 'n_estimators': 1000, ...  ...   \n",
       "\n",
       "   std_test_accuracy  rank_test_accuracy  split0_test_F1  split1_test_F1  \\\n",
       "2           0.002537                   1        0.788781        0.803516   \n",
       "3           0.005036                   2        0.785106        0.798503   \n",
       "1           0.007360                   3        0.789289        0.793782   \n",
       "7           0.003054                   4        0.783275        0.797217   \n",
       "6           0.003312                   5        0.774004        0.788307   \n",
       "\n",
       "   split2_test_F1  split3_test_F1  split4_test_F1  mean_test_F1  std_test_F1  \\\n",
       "2        0.795395        0.802655        0.787875      0.795645     0.006613   \n",
       "3        0.796936        0.801955        0.781604      0.792821     0.007975   \n",
       "1        0.803340        0.804890        0.772151      0.792690     0.011805   \n",
       "7        0.789154        0.801060        0.779098      0.789961     0.008237   \n",
       "6        0.779025        0.790700        0.773780      0.781164     0.007104   \n",
       "\n",
       "   rank_test_F1  \n",
       "2             1  \n",
       "3             2  \n",
       "1             3  \n",
       "7             4  \n",
       "6             5  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc.sort_values(by=[\"mean_test_F1\"], ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Ghosty/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "Y_1 = [dd_grid(dc,0.01, n) for n in np.unique(dc[\"param_n_estimators\"].values)]\n",
    "Y_2 = [dd_grid(dc,0.1, n) for n in np.unique(dc[\"param_n_estimators\"].values)]\n",
    "Z1 = np.vstack((Y_1, Y_2)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAGbCAYAAAAryfFSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df9xcdX3n/dfH/CBFKwkSLSaUpDWi6FK0EVxdq9XVhtoVt6s1sVuxq+XeVtot3lpxu7dSrFvtdkvVsvROK1V2LZFSrdm7UWRFbbcFN8GCGCIhxhQuwCI/K6UQEz73H3MGTiZzXdfkmjMzZ855PR+PeWTme87M+V6Pc3Kd9/X5fs+ZyEwkSZLa4gmT7oAkSdI4GX4kSVKrGH4kSVKrGH4kSVKrGH4kSVKrLJ50B+aydOnSXLZs2aS7IUljs3///kl3QRP2yCOP3J2ZK8e1vSc+8Yl58ODBSj7rkUceuTIzN1TyYSNU6/CzbNkyXvCCF0y6G5I0NjMzM5PugiZs9+7dfzfO7R08eJATTzyxks/avXv3cZV80Ig57CVJklrF8CNJNbJ69epJd0FqPMOPJElqFcOPJElqFcOPJElqFcOPJElqFcOPJElqFcOPJElqFcOPJElqFcOPJElqFcOPJNWMNzqURsvwI0mSWsXwI0mSWsXwI0mSWsXwI0mSWsXwI0mSxiIiNkTEzRGxJyLO67P8woi4vnjsjoj7S8s+GBFfLx5vKLV/ovjMr0fEJRGxZL5+GH4kSdLIRcQi4CLgDOBkYFNEnFxeJzPPzcxTM/NU4CPAp4r3vhp4PnAqcDrwzoh4cvG2TwDPAv4Z8H3AW+fri+FHkiSNw2nAnszcm5n7gS3AmXOsvwm4rHh+MvDlzDyQmf8I3ABsAMjMbVkA/g8w770iDD+SVEPe60dT6riI2FF6nF1atgq4rfR6pmg7TEScCKwFri6abgDOiIijI+I44MeBE3reswT4OeBz83Vy8aA/jSRJap6lS5dWFrZ37959d2aun2Vx9GnLWdbdCFyRmQcBMvPzEfEC4G+A7wDXAAd63vPfgL/MzL+ar59WfiRJ0jjMcGi1ZjVwxyzrbuTxIS8AMvP9xXygV9IJUrd0l0XEe4GVwNsH6YjhR5IkjcN2YF1ErI2IpXQCztbelSLiJGAFnepOt21RRDyleH4KcArw+eL1W4GfADZl5qODdGTe8FNcNnZXRHy9z7J3REQW429Ex4eLS9i+FhHPL617VkTcUjzOGqRzkiSpGTLzAHAOcCWwC7g8M3dGxAUR8ZrSqpuALcUE5q4lwF9FxE3AZuDfFp8H8AfA04Brikvk3zNfXwaZ8/Mx4PeBS8uNEXEC8Erg1lLzGcC64nE6cDFwekQcC7wXWE9nfO+6iNiamfcNsH1JktQAmbkN2NbT9p6e1+f3ed/DdK746veZRzx/ed7KT2b+JXBvn0UXAr/GoZOVzgQuLa44uxZYHhHH0ylHXZWZ9xaB5yqKS9QkSZLGaUFXexXlqdsz84aIQyZvz3YZ25Fc3nY2cDbAUUcdtZDuSZIkzeqIw09EHA38OvCqfov7tOUc7Yc3Zm6mM57Hk5/85NkugZMkSVqQhVzt9cN0bjx0Q0Tso3Op2lcj4geY/TK2I7m8TZKENzqURuWIw09m3piZT83MNZm5hk6weX5mfpvOJWtvKq76eiHwQGbeSWdm96siYkVErKBTNbqyuh9DkiRpMINc6n4ZnWvtT4qImYh4yxyrbwP2AnuAPwR+CSAz7wXeR+ca/+3ABUWbJEnSWM075yczN82zfE3peQJvm2W9S4BLjrB/kiRJlfIOz5IkqVUMP5IkqVUMP5IkqVUMP5IkqVUMP5JUY97rR6qe4UeSJLWK4UeSJLWK4UeSJLXKgr7VXZIkNcPSpUtbN7fMyo8kSWoVw48kSWoVw48kSWoVw48kSWoVw48k1VzbJqNKo2b4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kaQp4I0OpeoYfiRJUqsYfiRJ0lhExIaIuDki9kTEeX2WXxgR1xeP3RFxf2nZb0fEzojYFREfjogo2n80Im4sPvOx9rkYfiRJ0shFxCLgIuAM4GRgU0ScXF4nM8/NzFMz81TgI8Cnive+CHgxcArwXOAFwEuLt10MnA2sKx4b5uuL4UeSJI3DacCezNybmfuBLcCZc6y/CbiseJ7AMmApcBSwBPj7iDgeeHJmXpOZCVwKvHa+jhh+JElSVY6LiB2lx9mlZauA20qvZ4q2w0TEicBa4GqAzLwG+CJwZ/G4MjN3Fe+fGeQzyxYP/vNIkqSmWbp0KWvWrKnq4+7OzPWzLOs3FydnWXcjcEVmHgSIiGcAzwa6lz1eFRE/BvzTEXzmY2pd+Vm6dOmkuyBJkqoxA5xQer0auGOWdTfy+JAXwL8Grs3MBzPzQeCzwAuLzyzfB2Kuz3xMrcMPdO5t4f0tJMl7/WjqbQfWRcTaiFhKJ+Bs7V0pIk4CVgDXlJpvBV4aEYsjYgmdyc67MvNO4LsR8cLiKq83AZ+ZryO1Dz9dhiBJkqZXZh4AzgGuBHYBl2fmzoi4ICJeU1p1E7ClmMDcdQXwTeBG4Abghsz8n8WyXwT+CNhTrPPZ+foydXN+ugFoZmZmnjUlSVKdZOY2YFtP23t6Xp/f530Hgf9rls/cQefy94FNTeWnl5UgSZK0EFMbfroMQZIk6UhMffjpMgRJkqRBNCb8dBmCJEnSXBoXfroMQZIkqZ/Ghp8uQ5AkSSprfPjpMgRJagJ/j0nDa0346TIESZLUbq0LP12GIEmS2qm14afLECRJUru0Pvx0GYIkSWoHw08PQ5AkSc1m+JmFIUiSpGaaN/xExCURcVdEfL3U9l8i4hsR8bWI+HRELC8te3dE7ImImyPiJ0rtG4q2PRFxXvU/ymgYgiRJapZBKj8fAzb0tF0FPDczTwF2A+8GiIiTgY3Ac4r3/LeIWBQRi4CLgDOAk4FNxbpTwxAkqS78XSQNZ/F8K2TmX0bEmp62z5deXgu8rnh+JrAlMx8BvhURe4DTimV7MnMvQERsKda9aajeT0D3l87MzMyEeyJJ0vCWLFnCqlWrJt2Nsapizs+/Az5bPF8F3FZaNlO0zdZ+mIg4OyJ2RMSOhx9+uILujYaVIEmSptNQ4Scifh04AHyi29RntZyj/fDGzM2ZuT4z1y9btmyY7o2FIUiSpOky77DXbCLiLOCngFdkZjfIzAAnlFZbDdxRPJ+tvREcDpMkaTosqPITERuAdwGvycyHSou2Ahsj4qiIWAusA/4PsB1YFxFrI2IpnUnRW4frej1ZCZIkqd7mrfxExGXAy4DjImIGeC+dq7uOAq6KCIBrM/PfZ+bOiLiczkTmA8DbMvNg8TnnAFcCi4BLMnPnCH6e2rASJElSPQ1ytdemPs0fnWP99wPv79O+Ddh2RL1rAEOQJEn14h2ex8ThMEmS6sHwM2aGIElV8PeItHCGnwkxBEmSNBmGnwkzBEmSNF6Gn5owBEmSNB6Gn5oxBEmSNFqGn5oyBEmSNBqGn5ozBEmSVC3Dz5QwBEmSVA3Dz5QxBEmSplVEbIiImyNiT0Sc12f5hRFxffHYHRH3l5b9YER8PiJ2RcRNEbGmaH9FRHy1eM//johnzNcPw8+UMgRJ8neApklELAIuAs4ATgY2RcTJ5XUy89zMPDUzTwU+AnyqtPhS4L9k5rOB04C7ivaLgZ8t3vMnwH+ary+GnylnCJIkTYnTgD2ZuTcz9wNbgDPnWH8TcBlAEZIWZ+ZVAJn5YGY+VKyXwJOL58cAd8zXkXm/2FTTwS9QlSTVwHERsaP0enNmbi6erwJuKy2bAU7v9yERcSKwFri6aHomcH9EfKpo/1/AeZl5EHgrsC0i/gn4B+CF83XS8NMwhiBJ0pFYunRplSMId2fm+lmWRZ+2nGXdjcAVRbiBTl55CfA84Fbgk8CbgY8C5wI/mZlfiYh3Ar9LJxDNymGvhnI4TJJUMzPACaXXq5l9iGojxZBX6b1/WwyZHQD+HHh+RKwEfiQzv1Ks90ngRfN1xPDTcIYgSVJNbAfWRcTaiFhKJ+Bs7V0pIk4CVgDX9Lx3RRF2AF4O3ATcBxwTEc8s2l8J7JqvIw57tYTDYZKkScrMAxFxDnAlsAi4JDN3RsQFwI7M7AahTcCWzMzSew9GxDuAL0REANcBf1h85i8AfxYRj9IJQ/9uvr4YflrGECRJmpTM3AZs62l7T8/r82d571XAKX3aPw18+kj64bBXSzkcJjWD/4+lI2f4aTlDkCSpbQw/AgxBkqT2MPzoEIYgSVLTGX7UlyFIktRUhh/NyRAkSWoaw48GYgiSJDWF4UdHxBAkSZp2hh8tiCFIkjStDD8aiiFImjz/D0pHxvCjShiCJEnTwvCjShmCJEl1Z/jRSBiCJEl1ZfjRSBmCJEl1Y/jRWBiCJEl1sXjSHVC7dAPQzMzMhHsiSQJYvHgxK1eunHQ3xsrKjybCSpAkaVIMP5ooQ5AkadwMP6oFQ5A0HP//SIMz/KhWDEGSpFEz/KiWDEGSpFEx/KjWDEGSpKoZfjQVDEGSpKoYfjRVDEGSpGEZfjSVDEGSpIUy/GiqGYIkSUfK8KNGMARJ3utHGtS84SciLomIuyLi66W2YyPiqoi4pfh3RdEeEfHhiNgTEV+LiOeX3nNWsf4tEXHWaH4ctZ0hSJI0n0EqPx8DNvS0nQd8ITPXAV8oXgOcAawrHmcDF0MnLAHvBU4HTgPe2w1M0igYgiRJs5k3/GTmXwL39jSfCXy8eP5x4LWl9kuz41pgeUQcD/wEcFVm3puZ9wFXcXigkipnCJIk9VronJ+nZeadAMW/Ty3aVwG3ldabKdpmaz9MRJwdETsiYsfDDz+8wO5JhzIESZK6qp7wHH3aco72wxszN2fm+sxcv2zZsko7JxmA1HQe46qziNgQETcXc4PP67P8woi4vnjsjoj7S8sOlpZtLbVHRLy/WH9XRPzKfP1YvMD+/31EHJ+ZdxbDWncV7TPACaX1VgN3FO0v62n/0gK3LQ1l9erVzMzMTLobktQqEbEIuAh4JZ1csD0itmbmTd11MvPc0vq/DDyv9BH/lJmn9vnoN9PJHs/KzEcj4ql91jnEQis/W4HuFVtnAZ8ptb+pSGEvBB4ohsWuBF4VESuKic6vKtqkiXAYTE3msa2aOg3Yk5l7M3M/sIXOXOHZbAIuG+BzfxG4IDMfBcjMu+ZZf6BL3S8DrgFOioiZiHgL8AHglRFxC50E94Fi9W3AXmAP8IfALxUduRd4H7C9eFxQtEkT5UlCTeWxrQk5rjtvt3icXVp2JPN/TwTWAleXmpcVn3ltRLy21P7DwBuKZZ+NiHXzdXLeYa/M3DTLolf0WTeBt83yOZcAl8y3PWncHAZTU3lsaxBLlixh5cqVVX3c3Zm5fpZlA8//BTYCV2TmwVLbD2bmHRHxQ8DVEXFjZn4TOAp4ODPXR8RP08kaL5mrk97hWcK/kiVpDGabF9zPRnqGvDLzjuLfvXTmDXfnA80Af1Y8/zRwynwdMfxIBecBqYk8plUj24F1EbE2IpbSCThbe1eKiJOAFXSm3HTbVkTEUcXz44AXA92J0n8OvLx4/lJg93wdMfxIPTxZqGk8plUHmXkAOIfOBU+7gMszc2dEXBARrymtugnYUkyl6Xo2sCMibgC+CHygdJXYB4B/ExE3Ar8FvHW+viz0Unep0ZwrIUnVy8xtdC6OKre9p+f1+X3e9zfAP5vlM+8HXn0k/bDyI83Cv5bVJB7P0uMMP9IcPGGoSTyepQ7DjzQPJ0KrSTyWJcOPNDBPGpLUDIYf6QgYgNQEHsdqu1qHn6VLl7JmzZpJd0M6hCcONYHHsdqs1uGnywCkunEekJrAY1htNRXhBzoByBCkuvHkIUnTZ2rCT5chSHVjANI08/hVG01d+OkyAKlOPIFomnn8qm2mNvyAVSDVi/OANM08dtUmUx1+ugxBqhNPIpJUb40IP10GINWFAUjTyONWbdGo8ANWgVQfnkg0jTxu1QaNCz9dhiDVgfOAJKl+Fk+6A6O2Zs0a9u3bN+luqOVWr17NzMzMpLshDcTjtV0WLVrE8uXLJ92NsWps5afMKpDqwAqQponHq5qsFeGnyxCkSfOEomni8aqmalX46TIAaZI8oUjSZLUy/IBVIE2WE6E1LTxO1UStDT9dBiBNkicWTQOPUzVN68MPWAXSZHli0TTwOFWTGH5KDEGaFE8skjQ+hp8+DECaBOcBqe48PtUUhp9ZWAXSpHiCUZ15fKoJDD/zMARpEjzBSNLoGH4GZADSuBmAVFcem5p2hp8jYBVI4+Y8INWVx6WmmeFnAQxBGjdPNKojj0tNK8PPEAxAGidPNJJUDcPPkKwCaZwMQKobj0lNI8NPRQxBGhfnAaluPB41qIjYEBE3R8SeiDivz/ILI+L64rE7Iu7vWf7kiLg9In6/z3u3RsTXB+mH4adiBiCNiycc1YnHo+YTEYuAi4AzgJOBTRFxcnmdzDw3M0/NzFOBjwCf6vmY9wFf7vPZPw08OGhfDD8jYBVI4+IJR9IUOQ3Yk5l7M3M/sAU4c471NwGXdV9ExI8CTwM+X14pIp4EvB34zUE7YvgZIQOQxsEApLrwWNQ8VgG3lV7PFG2HiYgTgbXA1cXrJwD/FXhnn9XfVyx7aNCOLB50RS1MNwDt27dvov1Qs61evZqZmZlJd0PyWJxCixYt4phjjqnq446LiB2l15szc3PxPPqsn7N8zkbgisw8WLz+JWBbZt4W8fjHRMSpwDMy89yIWDNoJw0/Y2II0qh1/+r2xCNpgu7OzPWzLJsBTii9Xg3cMcu6G4G3lV7/c+AlEfFLwJOApRHxIPB3wI9GxD46meapEfGlzHzZXJ102GvMHArTqDn0oEnzGNQstgPrImJtRCylE3C29q4UEScBK4Brum2Z+bOZ+YOZuQZ4B3BpZp6XmRdn5tOL9n8B7J4v+IDhZyKcEK1R8+SjSfMYVK/MPACcA1wJ7AIuz8ydEXFBRLymtOomYEtmzjYkNjSHvSbIoTCNknMvNGkeg+qVmduAbT1t7+l5ff48n/Ex4GN92vcBzx2kH1Z+asAqkEbFGyJK0uGGCj8RcW5E7IyIr0fEZRGxrBjL+0pE3BIRnyzG9YiIo4rXe4rla6r4AZrCoTCNkgFIk+KxpzpacPiJiFXArwDrM/O5wCI6k5c+CFyYmeuA+4C3FG95C3BfZj4DuLBYTz0MQRoVT0KaFI891c2ww16Lge+LiMXA0cCdwMuBK4rlHwdeWzw/s3hNsfwVUb5YX4cwAGkUPAlpUjz2VCcLDj+ZeTvwO8CtdELPA8B1wP3FjG449O6Nj93ZsVj+APCUhW6/DawCaRScBySp7YYZ9lpBp5qzFng68EQ6X1bWq3up2kB3doyIsyNiR0TseOihge9U3WgGII2CAUjj5jGnuhhm2OtfAt/KzO9k5vfofPPqi4DlxTAYHHr3xsfu7FgsPwa4t/dDM3NzZq7PzPVHH330EN1rFqtAGgVPRho3jznVwTDh51bghRFxdDF35xXATcAXgdcV65wFfKZ4vrV4TbH86lHewKipDEGqmicjSW0zzJyfr9CZuPxV4MbiszYD7wLeHhF76Mzp+Wjxlo8CTyna3w6cN0S/W88ApCoZgDROHm+atKHu8JyZ7wXe29O8Fzitz7oPA68fZns6lHeIVpX8YlSNk3d/1iR5h+cGcChMVfKvco2Lx5omxfDTIAYgVcWTkqQmM/w0jFUgVcUApHHwONMkGH4ayhCkKnhDRI2Dx5jGzfDTcAYgVcGTk0bNY0zjNNTVXpoOXhWmKnh1jtRQB/bDvbdNuhdjZeWnRRwK07D861yj5PGlcTH8tJABSMNwHpBGyWNL42D4aSmrQBqWJylJ08rw03IGIA3DAKRR8LjSqBl+ZBVIQ/FEpVHwuNIoGX70GEOQFsp5QBoFjymNiuFHhzEAaaE8WUmaBoYf9WUVSAtlAFKVPJ40CoYfzckQpIXwhKUqeTypaoYfDcQApCPlCUtV8nhSlQw/GphVIB0pJ0JLqiPDj46YIUhHygCkKngcqSqGHy2YAUhHwhOXquBxpCoYfjQUq0A6Ep64JNWB4UeVMARpUM4D0rA8fjQsw48qZQDSoDyBaRgePxqG4UeVswqkQXkC0zA8fqZPRGyIiJsjYk9EnNdn+YURcX3x2B0R9xftJ0bEdUX7zoj490X70RHxFxHxjaL9A4P0w/CjkTEAaRCewKR2iIhFwEXAGcDJwKaIOLm8Tmaem5mnZuapwEeATxWL7gReVLSfDpwXEU8vlv1OZj4LeB7w4og4Y76+GH40UlaBNAjnAWmhPG6mymnAnszcm5n7gS3AmXOsvwm4DCAz92fmI0X7URT5JTMfyswvdtcBvgrMe1AsXvCPIB2BbgDat2/fRPuhelu9ejUzMzOT7oamjMfNkA7sJ++5rapPOy4idpReb87MzcXzVUB5QzN0qjiHiYgTgbXA1aW2E4C/AJ4BvDMz7+h5z3LgXwEfmq+Tta78LFmyhFWrVk26G6qQVSDNx7/ktRAeN7Vxd2auLz02l5ZFn/Vzls/ZCFyRmQcfWzHztsw8hU74OSsinvbYB0csplMl+nBm7p2vk7UOP12rVq0yBDWIQ2GajycyqZFmgBNKr1cDd8yy7kaKIa9eRcVnJ/CSUvNm4JbM/L1BOjIV4afLANQshiDNxQCkI+UxU3vbgXURsTYiltIJOFt7V4qIk4AVwDWlttUR8X3F8xXAi4Gbi9e/CRwD/OqgHZmq8ANWgZrIAKTZOBFaR8rjpb4y8wBwDnAlsAu4PDN3RsQFEfGa0qqbgC2ZWR4SezbwlYi4AfgynSu8boyI1cCv07l67KvFpfBvna8vUzvhuRuAbr/99gn3RFVwQrTm4oRWqRkycxuwraftPT2vz+/zvquAU/q0z9B/LtGcpq7y08sqULM4FKbZ+Be9BuWxovlMffgBh8KayACkfjypaVAeK5pLI8JPlyGoWawCqR/nAWlQHieaTaPCT5cBqFkMQOrHE5ukhWpk+AGrQE1jFUj9GIA0H48R9dPY8NNlCGoWQ5B6eXLTfDxG1Kvx4afLANQsBiCVOQ9I8/H4UFlrwg9YBWoaq0Dq5QlO0iBaFX66DEDNYghSmQFIs/HYUFcrww9YBWoiA5C6PMlpNh4bghaHny5DULNYBVKX84A0G48LtT78dBmAmsUQpC5PdJJ6GX5KrAI1jwFIYADS4Twm2s3w04chqFmsAgk82elwHhPttXjSHaizVatWcfvtt0+6G6pINwDt27dvov3Q5KxevZqZmZlJd0Oqldz/MN+7dfekuzFWQ1V+ImJ5RFwREd+IiF0R8c8j4tiIuCoibin+XVGsGxHx4YjYExFfi4jnV/MjjJZVoOaxCtRuToRWmcdCOw077PUh4HOZ+SzgR4BdwHnAFzJzHfCF4jXAGcC64nE2cPGQ2x4rQ1CzOBQmT3rq8lhonwWHn4h4MvBjwEcBMnN/Zt4PnAl8vFjt48Bri+dnApdmx7XA8og4fsE9nxADULMYgNrNk566PBbaZZjKzw8B3wH+OCL+NiL+KCKeCDwtM+8EKP59arH+KuC20vtnirZDRMTZEbEjInY8+OCDQ3RvdKwCNYtVoHbzpCe1zzDhZzHwfODizHwe8I88PsTVT/Rpy8MaMjdn5vrMXP+kJz1piO6NngGoWQxB7eU8IIFBuE2GCT8zwExmfqV4fQWdMPT33eGs4t+7SuufUHr/auCOIbZfC1aBmscA1F6e/OQx0A4LDj+Z+W3gtog4qWh6BXATsBU4q2g7C/hM8Xwr8Kbiqq8XAg90h8eawBDULFaB2suTnzwGmm/Y+/z8MvCJiFgK7AV+nk6gujwi3gLcCry+WHcb8JPAHuChYt3G8d5AzeK9gdrJ+wFJzTZU+MnM64H1fRa9os+6CbxtmO1Ni24FyBDUHGvWrDEAtUz3r39DUDsZgJvNr7cYIYfCmsWhsHZyCKS93PfNZfgZAwNQsxiC2seToNQshp8xsQrUPAagdjEAtZP7vZkMP2NmCGoWq0Dt4v2A2sl93jyGnwkxADWLIahdPBm2j/u8WQw/E2QVqHkMQO3hyVCaXoafGjAENYtVoPYwALWL+7s5DD81YgBqFgNQO3hCbBf393AiYkNE3BwReyLisO8DjYgLI+L64rE7Iu4v2k+NiGsiYmdEfC0i3lB6z9qI+EpE3BIRnyxuvDwnw0/NWAVqFqtA7eBE6HZxXy9MRCwCLgLOAE4GNkXEyeV1MvPczDw1M08FPgJ8qlj0EPCmzHwOsAH4vYhYXiz7IHBhZq4D7gPeMl9fDD81ZQBqFkNQO3hSlOZ0GrAnM/dm5n5gC3DmHOtvAi4DyMzdmXlL8fwOOl+avjIiAng5nS9XB/g48Nr5OjLsd3tphPyajObxazKaz69FaIcm7edH9+/nkdu+VdXHHRcRO0qvN2fm5uL5KuC20rIZ4PR+HxIRJwJrgav7LDsNWAp8E3gKcH9mHih95rzVA8PPFDAENYtfltp8TToxanbu577uzsx+3/kJEH3acpZ1NwJXZObBQz4g4njgvwNnZeajReVn0M98jMNeU8ShsGZxKKzZnAckHWYGOKH0ejVwxyzrbqQY8uqKiCcDfwH8p8y8tmi+G1geEd1izlyf+RjDz5RxQnTzGICazQDUbO7fI7IdWFdcnbWUTsDZ2rtSRJwErACuKbUtBT4NXJqZf9ptz8wEvgi8rmg6C/jMfB0x/EwpQ1CzWAVqNk+Qzeb+HUwxL+cc4EpgF3B5Zu6MiAsi4jWlVTcBW4pg0/UzwI8Bby5dCn9qsexdwNsjYg+dOUAfna8vzvmZcqtWrXIuUIM4H6i5nB/SbO7fwWTmNmBbT9t7el6f3+d9/wP4H7N85l46V5INzMpPA1gFah6rQM3kPCCpHgw/DWIIahaHwprLANRM7tfpYfhpIANQsxiCmskTZTO5X6eD4aehrAI1jwGoeTxRNpP7tf4MPw1nCGoWq0DN44lSGj/DT0sYgJrFANQsToRuHvdnvXmpe4uUA5CXx1z8moYAABfaSURBVE+/cgDy0vhm6J4wvWS6Gdyf9WX4aSmDULN4f6Bm8aTZLOUqkPu0HmodfpYuXeovgTEwCDWHIahZPGk2j+e0eqh1+CnrHT/1wBkNg1AzOCTWPJ40m8X9OVlTE356+RfR6PVOkjYMTSerQc3iSbNZPJdNxtSGnzKrQuNhVWi6GYKaxZNm83T36e7duyfck+ZrRPjpZRgaPYPQ9HJIrHmsBklHppHhp5d/IY2WQWh6WQ1qFkOQNJhWhJ8yq0KjZRCaToagZvEPPh2Jg4/s5/7dt026G2NV6/CzePHou2cYGh2D0PRxSKx5rAZJh6t1+AFYuXLlY8+/853vjHx7hqHRMAhNH6tBzWIIkh5X+/BTVg5CMP4w5C+NahiEposhqFkMQdKUhZ9e4w5DVoWqZxCaHg6JNYt/2KnNpjr89HKIbLoZhKaH1aBmsRqktmlU+ClziGy6GYSmgyGoWQxBaovGhp9eDpFNL4NQ/RmCmsU/5NR0rQk/vRwim05+31i9OS+oeawGqYlqHX6WLFnCypUrRx5OHCKbXlaF6stqULMYgtQktQ4/XeMOJw6RTSeDUD0ZgprFP9zUBFMRfnqNe8jKIbLpYxCqH4fEmsdqkKbVVIafsqZXhcC/tIZlEKofq0HNYgjStJn68NOr6WHIqtBwDEL1YghqFv9Q07RoXPjpNckhsnFs0zC0cAah+nBIrHmsBqnOhg4/EbEI2AHcnpk/FRFrgS3AscBXgZ/LzP0RcRRwKfCjwD3AGzJz37DbPxKTGLKa5Hwhf+kMziBUH1aDmsUQpDqqovLzH4BdwJOL1x8ELszMLRHxB8BbgIuLf+/LzGdExMZivTdUsP0Fc4hM/RiE6sEQ1Cz+YaY6ecIwb46I1cCrgT8qXgfwcuCKYpWPA68tnp9ZvKZY/opi/VktWrSI5cuXs3z58mG6ObCVK1c+9hj39saxzdWrVx/y0PxWrVr12EOTsWbNmsceagZ/B7VXRGyIiJsjYk9EnNdn+YURcX3x2B0R95eWnRURtxSPs0rtmyLixoj4WkR8LiKOm68fw1Z+fg/4NeD7i9dPAe7PzAPF6xmge9ZYBdwGkJkHIuKBYv27yx8YEWcDZ8Ohfyn0BqD777+fUXKITL2sCE2e1aBmcUisXYppMhcBr6STD7ZHxNbMvKm7TmaeW1r/l4HnFc+PBd4LrAcSuC4itgLfBT4EnJyZd0fEbwPnAOfP1ZcFh5+I+Cngrsy8LiJe1m3us2oOsOzxhszNwGaA5z3veYct72p6GHKIrN78mo3JMgQ1i3+ItcZpwJ7M3AsQEVvojArdNMv6m+gEHoCfAK7KzHuL914FbKAzkhTAEyPiHjpTcPbM15FhKj8vBl4TET8JLCs2+HvA8ohYXFR/VgN3FOvPACcAMxGxGDgGuHeI7R+iHIZGHYTAq8h0KKtCk+FVYs1jNWj8Dj7yPe775rer+rjjImJH6fXmoqgBpRGgwgxwer8PiYgTgbXA1XO8d1Vmfi8ifhG4EfhH4BbgbfN1csHhJzPfDby76OTLgHdk5s9GxJ8Cr6NzxddZwGeKt2wtXl9TLL86M2et7Ayj6VWh3m06RFYvBqHJsBrULIagqXV3Zq6fZdlAI0CFjcAVmXlwrvdGxBLgF+kMj+0FPkInm/zmXJ0cxX1+3gVsiYjfBP4W+GjR/lHgv0fEHjoVn40j2HZfTQ9DVoXqyyA0foagZjEENUp3BKirPDrUayOHVnBmgJf1vPdLwKkAmflNgIi4HDhsInWvSsJPZn6p6ATFWN5pfdZ5GHh9Fdsb1iSHyMAw1FYGofFySKxZrD43wnZgXXE/wNvpBJw39q4UEScBK+iMFHVdCfzniFhRvH4VnQrPMuDkiFiZmd+hM5l613wdqfUdnhctWjTybYy7KgTt+mJWf0n1ZxAaL6tBzWI1aDoVV3qfQyfILAIuycydEXEBsCMztxarbgK2lKfGZOa9EfE+OgEK4ILS5OffAP4yIr4H/B3w5vn6EiOadlOJ9evX5/bt2w9pe+CBB8bah3GEobJxhJNJbs9fVnMzCI2HIahZmvZ75eqrr75ujnkzlXvOscfkn/zLF1XyWaf+6efG2veFqnXlp59jjjnmsefjCEIOkVXLIbK5WREaDytBzWK1WUdq6sJPWTkIwejDkENk1fOX1uwMQqPnvKDmcUhMg5jq8NOr6WHIqlB7GYRGz2pQsxiCNJdGhZ9eDpFN9/YMQ/0ZhEbLENQsVpfVT6PDT1nTq0LgEFkbGYRGxyGx5rEapK7WhJ9eTQ9DVoXax+8bGx2rQc1iCFK9w8+B/XDvbXDsCfOvOySHyKZ7e4ahw1kVqp4hqFmsJrdXvcNP1723Hfp6xGGo6VUhcIisbQxC1XJIrHmsBrXLdISfXoahSlkVaheDULWsBjWLIagdpjP89CqHoYYPkYFhaFiGoccZhKpjCGoWq8fN1ozwU9bwqhBMdr6QQ2TNZRCqhkNizWM1qHmaF356NTwMWRWqllWhDoNQNawGNYshqDmaH356OURWKcNQ8xmEhmcIaharxdOvfeGnrOFVIXCIrGpt/6VnEBqOQ2LNYzVoOtU7/BzYT97zeECJp4y4UtPwMGRVqFptrwoZhIZjNahZpjkEHXj4APd8455Jd2Os6h1+eow1CIFDZBUzDDWXQWjhDEHNMs0hqE2mKvyUlYMQWBWqgkNk1WrrEJlBaGEcEmuWtv7/nxZTG356GYaqZVWoWm2tCvl9YwtjNahZrAbVT2PCT682DZGBYWjatmcYMggNwhDULIag+mhs+ClrelUI2vXFrA6RNYNBaHCGoGZp4//3umlF+OnV9DBkVWi6t9fGqpBBaDDOC2oeq0GTUevwk/sfHs92HCKrlGGoWm0LQwahwVgNapbe/+carVqHH4Dv3br7kNdLfvCZI91e06tC4BDZtG+vTWHIIDQ/Q5B05GoffnoZhqplVWi6twftmT9gEJqbQ2LS4KYu/PQqh6FRByFwiKxqhqFqtaUqZBCam9UgaW5TH37KrApVzyGy6d5eG8KQQWh2hiCpv0aFn16GoWpZFZru7UHzh8gMQv05JCYdqtHhp5dDZNUyDE339ppeFTII9Wc1SGpZ+CmzKlQ9h8ime3tNDkN+zcbhDEFqs1qHn0f37+eR274FwFEnrB3ptgxD1bIqNN3bg2YPkVkVepxDYmqjWoefsm4IgtEHIWjZEBkYhobU9DDUlqqQQWgNYAhS801N+CkrByGwKlSJCc4Xcohs+rbX1DBkEOowBGlUImID8CFgEfBHmfmBPuv8DHA+kMANmfnGov2DwKuL1d6XmZ8s2gP4TeD1wEHg4sz88Fz9mMrw08swVDGrQpVqelUImjlEZhBySEzViohFwEXAK4EZYHtEbM3Mm0rrrAPeDbw4M++LiKcW7a8Gng+cChwFfDkiPpuZ/wC8GTgBeFZmPtp9z1waEX56OURWMcNQpZoehppYFTIIWQ1SJU4D9mTmXoCI2AKcCdxUWucXgIsy8z6AzLyraD8Z+HJmHgAORMQNwAbgcuAXgTdm5qM975lVI8NPmVWhEXCIrFKTHCIbxzabFobaHoQMQc2zf/+j3H7Hd6v6uOMiYkfp9ebM3Fw8XwWUT1ozwOk9738mQET8NZ2hsfMz83PADcB7I+J3gaOBH+fx0PTDwBsi4l8D3wF+JTNvmauTjQ8/vQxDFbMqVKlJDFlNcr6QQWh6OSSmWdydmetnWRZ92rLn9WJgHfAyYDXwVxHx3Mz8fES8APgbOgHnGuBA8Z6jgIczc31E/DRwCfCSuTrZuvDTyyGyihmGKuUQ2fQwCBmCNK8ZOnNzulYDd/RZ59rM/B7wrYi4mU4Y2p6Z7wfeDxARfwLcUnrPnxXPPw388XwdqXX4OfjIfu7f/fjJdPkzR3sitSo0Ag6RVcohsunQ1iBkCNI8tgPrImItcDuwEXhjzzp/DmwCPhYRx9EZBttbTJZenpn3RMQpwCnA50vveTmdis9Lgd3Mo9bhp1c5CIFhaFgOkVWr6VWh3m06RDaYNgYhQ5D6ycwDEXEOcCWd+TyXZObOiLgA2JGZW4tlr4qIm+hctv7OIvAsozMEBvAPwL8tJj8DfAD4REScCzwIvHW+vkRm73BbfZxy/HH5/735Xw207qiDUK9xDJGVjWOIrNdYKkNlY6gMlY2jMlQ2jspQ2TjCySS3N61hqKstQajLIDS4Sy+99Lo55s1Ubu3i78sLlq+p5LPedM83xtr3hZqqys9crApVr01fzOoQWbXbG8c2p32IrG3fN2Y1SHXSmPDTyzBULYfIquUQWfWmfYisLcNjhiDVQWPDT69xTpyGdl1FBoahYTU9DFkVOjJtCEJeKq9Jak34KbMqVD2HyKrlEFm1pjkMtSkIGYI0LgsOPxFxAnAp8APAo3Tu4vihiDgW+CSwBtgH/Ezx/RxB58vMfhJ4CHhzZn51rm0cfOR73PfNb7Pih39god0ciGGoWlaFqtX0qlDvNh0im13Tg5AhSOMyTOXnAPB/Z+ZXI+L7gesi4io6XzD2hcz8QEScB5wHvAs4g86NitbRuZ31xRx+W+u+7vvmtx97PuogBA6RVc0wVK2mhyGrQoNpchBySEyjtuDwk5l3AncWz78bEbvofG/HmXRuSw3wceBLdMLPmcCl2bm2/tqIWB4RxxefM7ByEILRhyGrQtVziKxakxwiA8NQHbQhCBmCVKVK5vxExBrgecBXgKd1A01m3ln6avl+X2i2iiJAlT7rbOBsgOOPXjbvtg1D1XKIrFpNrwrBZOcLOUR2uKYGIUOQqjR0+ImIJ9H5To1fzcx/KO6+2HfVPm2H3WGx+PbXzQDPOfaYI74Do0Nk1TIMVavpYciqUL00MQg5JKYqDBV+ImIJneDzicz8VNH8993hrIg4HriraB/kC80qZVWoeq36YlaHyIZmGKqPJgchQ5CO1DBXewXwUWBXZv5uadFW4Cw637VxFvCZUvs5EbGFzkTnB450vs+wDEPVsipUraZXhcAhsrpoWhAyBOlIDVP5eTHwc8CNEXF90fYf6YSeyyPiLcCtwOuLZdvoXOa+h86l7j8/xLYr4RBZtQxD1Wp6GLIqVA9NCkIOiWlQtf5i05OOfmJe/IznAPCUZz1lrNseRxgq84tZq+WXslZr3F/KCn4xax1MexiC6QxBfrHp6E3NHZ7v+cY9h7wedRhyiKxaVoWq1fSqEDhEVgdNqAo5JKZ+pib89CqHoXFUhRwiq5ZhqFpND0MOkU3etAchh8Rm9/Cjj7Lru/sn3Y2xmtrwU2ZVqFpeRTYCLbqKDAxDw6p7GGpKEDIEtVcjwk8vw1C1HCKrWMOrQtCuL2Zt+xDZNAchQ1B7NTL89HKIrFoOkVWs4WHIqlC16lwVmtYgZAhqn1aEnzKrQtVyiGwEHCKrlGFoMqYxCDkvqD1qHX7273905NswDFXLIbKKNbwqBA6RVa2OQ2TTHIQMQc1U6/ADcPsd333s+aqnf//It+cQWbUMQxVreBiyKlStOlaFpi0IGYKaqfbhp6wchGD0YciqUPUmOV/IIbLhOUQ23durWxiapiDkkFizTFX46WUYqpZDZNWyKlQ9h8iqVachsmkMQoag6TXV4aeXQ2TVcoisWoahalkVqladqkLlIAT1DUOGoOnVqPBTZlWoWg6RVc8hsmoZhqpV1zBUxyDkkNj0aWz46WUYqpZDZNWyKlQ9h8iqVZchsmkJQoagemtN+OnlEFm1HCKrlmGoWlaFqlWXqlCdg5AhqN4iMyfdh1kd/4Sj8ueXrObZ3790rNsdRxgqG0cYKhtHGCobRxgqG8cQWdk4hsjKxjJEVjaGIbJe46gMlY2jMlQ2jkrNJLc36cnTdQtCXYMGoUsvvfS6zFw/2t48rnuurcJv7d871r4v1FSEn16GoWoZhqplGKqWQWi6t2cQOtx8IcjwM3pTGX7KDELVanoQAsNQ5QxDlTIMjU7dgtBsIcjwM3pTH356GYaq1fQwZBCqmENklTMMjUadg5DhZ/QaF356GYaqZRiqlmGoWgah6d4eTCYM1SkI7du3r9HhJyI2AB8CFgF/lJkf6LPOzwDnAwnckJlvLNp/G3g18ATgKuA/ZCnERMRW4Icy87nz9bPxV3vt+u7+x56PIwh5FVm1vKS+Wl5FVi2vIqveJC6pr9NVY+V7BjVNRCwCLgJeCcwA2yNia2beVFpnHfBu4MWZeV9EPLVofxHwYuCUYtX/DbwU+FKx/KeBBwftS+PDT1k5CMHow5D3FqreJC+p90aLFZjgjRbBMDRt25vEJfV1CkINdBqwJzP3AkTEFuBM4KbSOr8AXJSZ9wFk5l1FewLLgKVAAEuAvy8+50nA24GzgcsH6Uitw8+3c//dv7V/79+NbAP3zL9KRY4D7h7j9jpuHPP2pkNnX6gO3Bf14v6oj5PGubFv5/4rf2v/3uMq+rhlEbGj9HpzZm4unq8CyuXgGeD0nvc/EyAi/prO0Nj5mfm5zLwmIr4I3Ekn/Px+Zu4q3vM+4L8CDw3ayVqHn8xcOf9a9RcRO6ZhAlgbuC/qw31RL+6P+ugJDyOXmRvGtKnot/me14uBdcDLgNXAX0XEc+mE82cXbQBXRcSPAf8APCMzz42INYN2pNbhR5IkNcYMUB7vXg3c0WedazPze8C3IuJmHg9D12bmgwAR8VnghcB3gR+NiH10Ms1TI+JLmfmyuTryhKF/FEmSpPltB9ZFxNqIWApsBLb2rPPnwI8DRMRxdIbB9gK3Ai+NiMURsYTOZOddmXlxZj49M9cA/wLYPV/wAcPPuGyefxWNifuiPtwX9eL+qI9G7ovMPACcA1wJ7AIuz8ydEXFBRLymWO1K4J6IuAn4IvDOzLwHuAL4Jp3ZrDfQuQT+fy60L7W+z48kSVLVrPxIkqRWMfxIkqRWMfwMKSI2RMTNEbEnIs7rs/yoiPhksfwr3UvxIuKVEXFdRNxY/Pvycfe9aRa6L0rLfzAiHoyId4yrz001zL6IiFMi4pqI2Fn8/1g2zr43zRC/o5ZExMeLfbArIt497r43zQD74sci4qsRcSAiXtez7KyIuKV4nDW+XjdUZvpY4IPODZi+CfwQnbtO3gCc3LPOLwF/UDzfCHyyeP484OnF8+cCt0/655nmxzD7orT8z4A/Bd4x6Z9nmh9D/r9YDHwN+JHi9VOARZP+mab1MeS+eCOwpXh+NLAPWDPpn2laHwPuizV0vr7hUuB1pfZj6VzxdCywoni+YtI/0zQ/rPwM57FbdWfmfqB7q+6yM4GPF8+vAF4REZGZf5uZ3fsb7KRzV8yjxtLrZlrwvgCIiNfS+YWyc0z9bbJh9sWrgK9l5g0AmXlPZh4cU7+baJh9kcATI2Ix8H3Afjo3lNPCzLsvMnNfZn4NeLTnvT8BXJWZ92bnax+uAsZ1Y8JGMvwMp9+tulfNtk52LvN7gM5fs2X/BvjbzHxkRP1sgwXvi4h4IvAu4DfG0M82GOb/xTOBjIgri/L/r42hv002zL64AvhHOl8ncCvwO5l576g73GCD7ItRvFd9eIfn4Qxyq+4514mI5wAfpPMXrxZumH3xG8CFmflgUQjScIbZF4vp3KjsBXS+p+cLEXFdZn6h2i62xjD74jTgIPB0OkMtfxUR/yuLL6XUERtkX4ziverDys9wBr1V9wkARfn4GODe4vVq4NPAmzLzmyPvbbMNsy9OB367uD36rwL/MSLOGXWHG2yYfTEDfDkz787Mh4BtwPNH3uPmGmZfvBH4XGZ+LzvfrP3XgN/9tXCD7ItRvFd9GH6GM8iturcC3Zn5rwOuzsyMiOXAXwDvzsy/HluPm2vB+yIzX5KZa7Jze/TfA/5zZv7+uDreQAveF3Tu7npKRBxdnIhfCtw0pn430TD74lbg5dHxRDrfo/SNMfW7iQbZF7O5EnhVRKyIiBV0RgquHFE/W8HwM4Qc7FbdH6Uzr2QP8Hage3njOcAzgP8nIq4vHk8d84/QGEPuC1VomH1RTOb8XToniuuBr2bmX4z7Z2iKIf9fXAQ8Cfg6nf3xx8VkXC3AIPsiIl4QETPA64H/NyJ2Fu+9F3gfnf2wHbjA+VfD8estJElSq1j5kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrWL4kSRJrfL/A/koYUxMCtBiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.contourf(LR, NE, Z1, 20, cmap='RdGy')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "dr = pd.read_csv(\"Hyperparam_search/Graident_Tree_Boosting_LSTMVAE_LS_GS_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>param_n_iter_no_change</th>\n",
       "      <th>param_subsample</th>\n",
       "      <th>params</th>\n",
       "      <th>...</th>\n",
       "      <th>std_test_accuracy</th>\n",
       "      <th>rank_test_accuracy</th>\n",
       "      <th>split0_test_F1</th>\n",
       "      <th>split1_test_F1</th>\n",
       "      <th>split2_test_F1</th>\n",
       "      <th>split3_test_F1</th>\n",
       "      <th>split4_test_F1</th>\n",
       "      <th>mean_test_F1</th>\n",
       "      <th>std_test_F1</th>\n",
       "      <th>rank_test_F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>72.628706</td>\n",
       "      <td>16.228849</td>\n",
       "      <td>0.296876</td>\n",
       "      <td>0.060181</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1000, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004295</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766551</td>\n",
       "      <td>0.784310</td>\n",
       "      <td>0.776741</td>\n",
       "      <td>0.785773</td>\n",
       "      <td>0.773225</td>\n",
       "      <td>0.777320</td>\n",
       "      <td>0.007119</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>426.715195</td>\n",
       "      <td>17.676517</td>\n",
       "      <td>1.677381</td>\n",
       "      <td>0.042079</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1500, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003454</td>\n",
       "      <td>3</td>\n",
       "      <td>0.770459</td>\n",
       "      <td>0.780865</td>\n",
       "      <td>0.776847</td>\n",
       "      <td>0.781425</td>\n",
       "      <td>0.766778</td>\n",
       "      <td>0.775275</td>\n",
       "      <td>0.005778</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>60.229023</td>\n",
       "      <td>19.307404</td>\n",
       "      <td>0.246718</td>\n",
       "      <td>0.071638</td>\n",
       "      <td>0.10</td>\n",
       "      <td>500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 500, 'n...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004450</td>\n",
       "      <td>2</td>\n",
       "      <td>0.769641</td>\n",
       "      <td>0.784327</td>\n",
       "      <td>0.771124</td>\n",
       "      <td>0.782359</td>\n",
       "      <td>0.768664</td>\n",
       "      <td>0.775223</td>\n",
       "      <td>0.006705</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>54.951538</td>\n",
       "      <td>21.385327</td>\n",
       "      <td>0.223219</td>\n",
       "      <td>0.077701</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1500</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.1, 'n_estimators': 1500, '...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004027</td>\n",
       "      <td>4</td>\n",
       "      <td>0.763158</td>\n",
       "      <td>0.781094</td>\n",
       "      <td>0.771697</td>\n",
       "      <td>0.781519</td>\n",
       "      <td>0.768503</td>\n",
       "      <td>0.773194</td>\n",
       "      <td>0.007165</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>285.818368</td>\n",
       "      <td>13.736511</td>\n",
       "      <td>1.138197</td>\n",
       "      <td>0.013024</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1000</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7</td>\n",
       "      <td>{'learning_rate': 0.01, 'n_estimators': 1000, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004276</td>\n",
       "      <td>5</td>\n",
       "      <td>0.764572</td>\n",
       "      <td>0.782514</td>\n",
       "      <td>0.773292</td>\n",
       "      <td>0.778818</td>\n",
       "      <td>0.762343</td>\n",
       "      <td>0.772308</td>\n",
       "      <td>0.007832</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "2           2      72.628706     16.228849         0.296876        0.060181   \n",
       "7           7     426.715195     17.676517         1.677381        0.042079   \n",
       "1           1      60.229023     19.307404         0.246718        0.071638   \n",
       "3           3      54.951538     21.385327         0.223219        0.077701   \n",
       "6           6     285.818368     13.736511         1.138197        0.013024   \n",
       "\n",
       "   param_learning_rate  param_n_estimators  param_n_iter_no_change  \\\n",
       "2                 0.10                1000                      20   \n",
       "7                 0.01                1500                      20   \n",
       "1                 0.10                 500                      20   \n",
       "3                 0.10                1500                      20   \n",
       "6                 0.01                1000                      20   \n",
       "\n",
       "   param_subsample                                             params  ...  \\\n",
       "2              0.7  {'learning_rate': 0.1, 'n_estimators': 1000, '...  ...   \n",
       "7              0.7  {'learning_rate': 0.01, 'n_estimators': 1500, ...  ...   \n",
       "1              0.7  {'learning_rate': 0.1, 'n_estimators': 500, 'n...  ...   \n",
       "3              0.7  {'learning_rate': 0.1, 'n_estimators': 1500, '...  ...   \n",
       "6              0.7  {'learning_rate': 0.01, 'n_estimators': 1000, ...  ...   \n",
       "\n",
       "   std_test_accuracy  rank_test_accuracy  split0_test_F1  split1_test_F1  \\\n",
       "2           0.004295                   1        0.766551        0.784310   \n",
       "7           0.003454                   3        0.770459        0.780865   \n",
       "1           0.004450                   2        0.769641        0.784327   \n",
       "3           0.004027                   4        0.763158        0.781094   \n",
       "6           0.004276                   5        0.764572        0.782514   \n",
       "\n",
       "   split2_test_F1  split3_test_F1  split4_test_F1  mean_test_F1  std_test_F1  \\\n",
       "2        0.776741        0.785773        0.773225      0.777320     0.007119   \n",
       "7        0.776847        0.781425        0.766778      0.775275     0.005778   \n",
       "1        0.771124        0.782359        0.768664      0.775223     0.006705   \n",
       "3        0.771697        0.781519        0.768503      0.773194     0.007165   \n",
       "6        0.773292        0.778818        0.762343      0.772308     0.007832   \n",
       "\n",
       "   rank_test_F1  \n",
       "2             1  \n",
       "7             2  \n",
       "1             3  \n",
       "3             4  \n",
       "6             5  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr.sort_values(by=[\"mean_test_F1\"], ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Ghosty/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "Y_1 = [dd_grid(dr,0.01, n) for n in np.unique(dr[\"param_n_estimators\"].values)]\n",
    "Y_2 = [dd_grid(dr,0.1, n) for n in np.unique(dr[\"param_n_estimators\"].values)]\n",
    "Z2 = np.vstack((Y_1, Y_2)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAGfCAYAAACwWLNEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df/RkdX3n+efb7gYiCjRBoaEJdBQxTGIC6QEzWWeMrAYzWXF3Nek2x2Ciy9lRzIyuiTg5qwaTPeYnMerodgITneOhMfir56QjMmKSSRYdWiUqEKElRL78EAkNCRIbunnvH3ULiur6/qi6Vffn83HO9/S37r1Vdb9Var18vT9VFZmJJElSXzyl7hOQJEmqkuFHkiT1iuFHkiT1iuFHkiT1iuFHkiT1iuFHkiT1iuFHkiRVIiLOi4ivR8TeiLh4wv5LI+KG4ueWiHig2P4TI9tviIjvRsTLi33nRsSXiu1/FRHPXvU8/JwfSZK0aBGxDrgFeDGwBFwPbM/Mm5Y5/o3AmZn5i2PbjwX2Apsz8+GIuAU4PzNvjojXA2dn5mtWOhebH0mSVIWzgb2ZeVtmPgLsBM5f4fjtwBUTtr8C+LPMfLi4nMBRxe9HA3etdiLr13zKNdiwYUMeccQRdZ+GpI7ZsGFD3acgLWvfvn33ZeYzqrq/I488Mg8ePDiX29q/f/+NwHdHNu3IzB3F7ycBd4zsWwLOmXQ7EXEKsAW4dsLubcDvjVx+HbA7Iv4Z+Efg+audZ6PDzxFHHMFZZ51V92lI6olNmzbVfQoSV1555d9XeX8HDx7klFNOmctt3XLLLd/NzK3L7I4J25Zbe7MNuCozn5TKImIT8EPA1SOb3wT8VGZ+ISJ+mUEwet1K59no8CNJVbr77ruX3WcwkkpbAk4eubyZ5UdU24A3TNj+M8AnMvNRgIh4BvDDmfmFYv+VwKdXOxHDjyStwXLByFAkrdn1wGkRsQW4k0HAedX4QRFxOrARuG7CbWwH3jZyeR9wdEQ8JzOHi6lvXu1EDD+SVIJtkbQ2mXkgIi5iMLJaB1yemTdGxCXAnszcVRy6HdiZY29Hj4hTGTRHfzF2m/8H8LGIeIxBGHrSu8MmMfxI0oLYFklPlpm7gd1j294+dvmdy1z3dgaLpse3fwL4xDTnYfiRpIrZFkn1MvxIUoMYjKTFM/xIUks4RpPmw/AjSS1nWyRNx/AjSR1mWyQdyvAjST1kW6Q+M/xIkp7EtkhdZ/iRJK2JbZG6wvAjSSrNtkhtYviRJC2MbZGayPAjSaqFbZHqYviRJDWKbZEWzfAjSWoN2yLNg+FHktR6tkWz27BhAyeccMJcbuuWW26Zy+0smuFHktRptkUaZ/iRJPWSbVF/GX4kSRpjW9Rthh9JktbItqgbnrLaARFxeUTcGxFfm7DvLRGREXFccTki4g8iYm9EfCUizho59oKIuLX4uWC+f4YkSfW6++67J/6oedbS/Pwx8D7gw6MbI+Jk4MXAN0c2vxQ4rfg5B/gAcE5EHAu8A9gKJPDFiNiVmfvK/gGSJDWZbVHzrBp+MvMvI+LUCbsuBX4F+NTItvOBD2dmAp+PiGMiYhPwQuCazLwfICKuAc4Drih19pIktZjNUD1WHXtNEhEvA+7MzL8Z23UScMfI5aVi23LbJ932hRGxJyL2PProo7OcniRJ0rKmXvAcEU8FfhV4yaTdE7blCtsP3Zi5A9gB8PSnP33iMZIkSbOapfl5FrAF+JuIuB3YDHwpIk5g0OicPHLsZuCuFbZLkiRVaurwk5lfzcxnZuapmXkqg2BzVmbeA+wCfr5419fzgQcz827gauAlEbExIjYyaI2unt+fIUmStDZreav7FcB1wOkRsRQRr13h8N3AbcBe4A+B1wMUC53fBVxf/FwyXPwsSZJUpbW822v7KvtPHfk9gTcsc9zlwOVTnp8kSdJczfRuL0mSpLYy/EiSpF4x/EiSpF4x/EiSpF7xW90lSeqxDRs29O47xmx+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSrxh+JElSJSLivIj4ekTsjYiLJ+y/NCJuKH5uiYgHiu0/MbL9hoj4bkS8vNj3keI2vxYRl0fEhtXOw/AjSZIWLiLWAe8HXgqcAWyPiDNGj8nMN2Xmj2TmjwDvBT5ebP/cyPYXAQ8Dnymu9hHgucAPAd8DvG61czH8SJKkKpwN7M3M2zLzEWAncP4Kx28Hrpiw/RXAn2XmwwCZuTsLwP8ANq92IuunPnVJktQZhx12GJs3r5oX1uq4iNgzcnlHZu4ofj8JuGNk3xJwzqQbiYhTgC3AtRN2bwN+b8J1NgCvBv79aidp+JEkSfNyX2ZuXWZfTNiWyxy7DbgqMw8+6QYiNjEYb1094Tr/CfjLzPzvq52kYy9JklSFJeDkkcubgbuWOXYbk0dePwN8IjMfHd0YEe8AngG8eS0nYviRJElVuB44LSK2RMRhDALOrvGDIuJ0YCNw3YTbOGQdUES8DvhJYHtmPraWE2l0+NmwYdV3q0mSpBbIzAPARQxGVjcDH83MGyPikoh42cih24GdxQLmx0XEqQyao78Yu+kPAscD1xVvg3/7aufS+DU/mzZtAuDuu++u+UwkSVIZmbkb2D227e1jl9+5zHVvZ7Boenz71Fmm0c3PqE2bNj0ehCRJkmbVmvAzZAiSJElltC78DBmAJEnSLFobfsAWSJIkTa/V4WfIECRJktaqE+FnyBAkSZJW06nwM2QIkiRJy+lk+BkyBEmSpHGdDj9DhiBJkjTUi/AzZAiSJEm9Cj9DhiBJkvqrl+FnyBAkSVL/NP6LTavgl6dKkvpqw4YNnHjiiXWfRqV63fyMswmSJKn7DD8TGIIkSeouw88KDEGSJHWP4WcNDEGSJHWH4WcKhiBJktpv1fATEZdHxL0R8bWRbb8dEX8bEV+JiE9ExDEj+94WEXsj4usR8ZMj288rtu2NiIvn/6dUxxAkSVJ7raX5+WPgvLFt1wA/mJnPA24B3gYQEWcA24B/UVznP0XEuohYB7wfeClwBrC9OLbVDEGSJLXPquEnM/8SuH9s22cy80Bx8fPA5uL384Gdmbk/M/8O2AucXfzszczbMvMRYGdxbCcYgCRJao95rPn5ReDPit9PAu4Y2bdUbFtu+yEi4sKI2BMRe/bv3z+H06uGLZAkSe1QKvxExK8CB4CPDDdNOCxX2H7oxswdmbk1M7cefvjhZU6vFoYgSZKabeavt4iIC4CfBs7NzGGQWQJOHjlsM3BX8fty2zvJr8yQJKmZZmp+IuI84K3AyzLz4ZFdu4BtEXF4RGwBTgP+B3A9cFpEbImIwxgsit5V7tTbwSZIkqRmWbX5iYgrgBcCx0XEEvAOBu/uOhy4JiIAPp+Z/2dm3hgRHwVuYjAOe0NmHixu5yLgamAdcHlm3riAv6exbIIkSWqGVcNPZm6fsPmyFY7/DeA3JmzfDeye6uw6yBAkSVK9/ITnmjgOkySpHoafmhmCJEmqluGnIQxBkiRVY+a3umsxXBMkSarShg0bOPHEE+s+jUrZ/DSUTZAk9c8999xT9yn0gs1Pw9kESVK3GXiqZ/hpCUOQJHWHgadehp+WMQRJUnsZeprB8NNShiBJagcDT/MYflrOECRJzWPgaTbDT0cYgiSpXgae9vCt7h3j2+MlqVr33HOPwadlbH46yBZIkhbLsNNuhp8OMwRJ0vwYeLrD8NMDhiBJmp2hp3sMPz1iCJKktTHwdJvhp4cMQZJ0KANPfxh+eswQJEmGnj7yre7yG+Ql9c7w7ekGn2pFxHkR8fWI2BsRF0/Yf2lE3FD83BIRD4zs+76I+ExE3BwRN0XEqcX2iIjfKI6/OSJ+abXzsPnR42yCJHWZQadeEbEOeD/wYmAJuD4idmXmTcNjMvNNI8e/EThz5CY+DPxGZl4TEU8DHiu2vwY4GXhuZj4WEc9c7VwMPzqEIUhSVxh4GuVsYG9m3gYQETuB84Gbljl+O/CO4tgzgPWZeQ1AZj40cty/A16VmY8V++5d7UQMP1qWIUhSWxl61m7Dhg0cf/zx87q54yJiz8jlHZm5o/j9JOCOkX1LwDmTbiQiTgG2ANcWm54DPBARHy+2/zfg4sw8CDwL+NmI+F+BbwO/lJm3rnSShh+tyhAkqQ0MPI1wX2ZuXWZfTNiWyxy7DbiqCDcwyCsvYDAG+yZwJYNx12XA4cB3M3NrRPxvwOXFsctywbPWzIXRkprGhcutssRgbc7QZuCuZY7dBlwxdt0vZ+ZtmXkA+CRw1si+jxW/fwJ43monYvjR1AxBkupm4Gml64HTImJLRBzGIODsGj8oIk4HNgLXjV13Y0Q8o7j8Ip5YK/TJ4jLAvwFuWe1EHHtpZo7DJFXJsNNumXkgIi4CrgbWAZdn5o0RcQmwJzOHQWg7sDMzc+S6ByPiLcBnIyKALwJ/WOx+N/CRiHgT8BDwutXOxfCj0gxBkhbFwNMtmbkb2D227e1jl9+5zHWvYcJIKzMfAP7tNOdh+NHcGIIkzYOBR4tm+NHcGYIkzcLQo6oYfrQwmzZtMgBJWpGBR3Uw/GihbIEkjTPwqG6GH1XCECTJ0KOmMPyoUoYgqV8MPGoiw49qYQiSusvAo6Yz/KhWhiCpOww9agvDjxrBECS1k4FHbWT4UaMYgqTmM/Co7Qw/aiRDkNQsBh51ieFHjWYIkupl6FEXGX7UCoYgqToGHnWd4UetYgiSFsPA01/r16/n2GOPrfs0KmX4USsZgqT5MPSojww/ajVDkDQ9A4/6zvCjTjAESSsz8EhPMPyoUwxB0pMZeqRDPWW1AyLi8oi4NyK+NrLt2Ii4JiJuLf7dWGyPiPiDiNgbEV+JiLNGrnNBcfytEXHBYv4caWDTpk2PByGpb+65557HfyQdatXwA/wxcN7YtouBz2bmacBni8sALwVOK34uBD4Ag7AEvAM4BzgbeMcwMEmLZAhSXxh4pLVbNfxk5l8C949tPh/4UPH7h4CXj2z/cA58HjgmIjYBPwlck5n3Z+Y+4BoODVTSwhiA1EUGHmk2a2l+Jjk+M+8GKP59ZrH9JOCOkeOWim3LbT9ERFwYEXsiYs/+/ftnPD3pULZAkiSYPfwsJyZsyxW2H7oxc0dmbs3MrYcffvhcT04CQ5C644QTTqj7FKRWmjX8fKsYZ1H8e2+xfQk4eeS4zcBdK2yXamMIUhcYgKTpzRp+dgHDd2xdAHxqZPvPF+/6ej7wYDEWuxp4SURsLBY6v6TYJtXOEKS2O+GEEwxB0hTW8lb3K4DrgNMjYikiXgu8G3hxRNwKvLi4DLAbuA3YC/wh8HqAzLwfeBdwffFzSbFNagwDkNrOACStzaofcpiZ25fZde6EYxN4wzK3czlw+VRnJ1XMD0lU2w0DkO8Ak5Y37wXPUic4ClPb2QJJy2t0+DnssMPqPgX1nCFIbWYAkiZr/Hd7bd68+fHfl5aWajwT9dmmTZschamVHINJh2p08zNu8+bNTwpDUpVsgdRmtkDSExrf/EwyDEA2QaqDi6LVVieccIINkA6xbt06Nm7s19dttqr5GWcTpDrZBKmN/EwgqaXNzzjXBalONkFqI1sg9Vmrm59JbINUF1sgtY0NkPqqc+FnyBCkOjgKU9s4BlMfdWLstRJHYqqDozC1jWMw9Ulnm59JbINUNZsgtYktkPqiV+FnyBCkqhmA1CYGIHVdL8PPkCFIVbIFUpsYgNRlnV/zsxauC1KVXA+ktvCrMdRVvW5+JrENUlVsgtQWtkDqGsPPMgxBqooBSG1gAFKXOPZahSMxVcFRmNrAMZi6wuZnCrZBWjRHYWoDWyC1neFnBoYgLZohSE3nZwJpFhFxXkR8PSL2RsTFE/ZfGhE3FD+3RMQDI/u+LyI+ExE3R8RNEXFqsX1LRHwhIm6NiCsj4rDVzsPwU4IhSItmAFLTGYC0VhGxDng/8FLgDGB7RJwxekxmvikzfyQzfwR4L/Dxkd0fBn47M38AOBu4t9j+m8ClmXkasA947WrnYviZg2EIMghpEWyB1HQGIK3R2cDezLwtMx8BdgLnr3D8duAKgCIkrc/MawAy86HMfDgiAngRcFVxnQ8BL1/tRAw/c2YI0qIYgtRkjsFUOC4i9oz8XDiy7yTgjpHLS8W2Q0TEKcAW4Npi03OAByLi4xHx5Yj47aJJ+l7ggcw8sNptjvLdXgsyDEC+Q0zz5jvD1GR+QWr7rFu3jqOPPnpeN3dfZm5dZl9M2JbLHLsNuCozDxaX1wMvAM4EvglcCbwG2DXFbT7O5mfBHIlpUWyB1FQ2QFrGEnDyyOXNwF3LHLuNYuQ1ct0vFyOzA8AngbOA+4BjImJY5qx0m48z/FTIEKR5cxSmpnIMpgmuB04r3p11GIOAc0hzExGnAxuB68auuzEinlFcfhFwU2Ym8DngFcX2C4BPrXYihp8aGII0b4YgNZUBSENFY3MRcDVwM/DRzLwxIi6JiJeNHLod2FkEm+F1DwJvAT4bEV9lMEL7w2L3W4E3R8ReBmuALlvtXFzzUyPXBWneXA+kJvKToTWUmbuB3WPb3j52+Z3LXPca4HkTtt/G4J1ka2bz0wCuC9K82QKpiWyB1BSGn4YxBGleHIWpiQxAagLDT0MZgjQvhiA1jYuhVTfX/DSc3yqveXE9kJrGzwRSXWx+WsQ2SPNgC6QmsQFSHQw/LWQIUlmOwtQkjsFUNcNPixmCVJYhSE1iAFJVXPPTAa4LUlmuB1JT+JlAqoLNT8fYBqkMWyA1hS2QFsnw01GGIM3KUZiawgCkRXHs1XGOxDQrR2FqAsdgWgSbnx6xDdIsbILUBLZAmifDTw8ZgjQLQ5DqZgDSvDj26jG/VV6z2LRpk6Mw1cYx2PytW7eOo446qu7TqJThR64L0tRcD6S6+dUYKqPRY68NGzZw4okn1n0aveJITNNwFKY6+cnQmlWjw8/QiSeeaAiqmCFI0zAEqU4GIE2rVWOv0QB011131Xgm/eFITNNwPZDq4hhM0yjV/ETEmyLixoj4WkRcERFHRMSWiPhCRNwaEVdGxGHFsYcXl/cW+08tc9+2QdWzDdJa2AKpLo7BtFYzh5+IOAn4JWBrZv4gsA7YBvwmcGlmngbsA15bXOW1wL7MfDZwaXFcaYag6hmCtBaGINXFAKTVlF3zsx74nohYDzwVuBt4EXBVsf9DwMuL388vLlPsPzciouT9P84QVD1DkNbCEKQ6GIC0kpnX/GTmnRHxO8A3gX8GPgN8EXggMw8Uhy0BJxW/nwTcUVz3QEQ8CHwvcN+s5zCJ64Kq57ogrYXrgVQ1PxNIyykz9trIoM3ZApwIHAm8dMKhObzKCvtGb/fCiNgTEXu+853vzHp6gG1QHWyDtBJbINXBFkjjyoy9/mfg7zLz25n5KPBx4F8BxxRjMIDNwLB+WQJOBij2Hw3cP36jmbkjM7dm5tYjjzyyxOk9wRBUPUOQVmIIUtVcDK1RZcLPN4HnR8RTi7U75wI3AZ8DXlEccwHwqeL3XcVliv3XZuYhzc8iDUOQQag6wxBkENIkhiBVzQAkKBF+MvMLDBYufwn4anFbO4C3Am+OiL0M1vRcVlzlMuB7i+1vBi4ucd6lGYKqZwjScgxAqpIBSKU+5DAz3wG8Y2zzbcDZE479LvDKMve3CMMA5OLo6viFqprE7wtTlVwM3W+t+HqLKtgEVc8mSJM4ClOVbIH6qVVfb1EF3ypfPd8qr0lsglQVvxqjf2x+VmAbVD3bII2zBVIVfDdYvxh+1sAQVD1DkEY5ClNVDED94NhrCo7EqudITKMchakKvVsMfeBR2Nev1zSbnxnZBlXPNkhDNkGqgi1Qdxl+SjIEVc8QpCEDkBbNANRNhp85MQRVzxAksAXS4rkYuntc8zNnrguqnuuCBK4H0uL5lvjusPlZINug6tkGySZIi2QD1A2GnwoYgqpnCJIBSIviGKz9HHtVyJFY9RyJ9ZujMC2SY7D2svmpiW1Q9WyD+stRmBbFFqidDD81MwRVzxDUX4YgLYoBqF0MPw1hCKqeIai/DEBaBANQe7jmp2FcF1Q91wX1k+uBtAi9+2qMlrL5aTDboOrZBvWPozAtgi1Qsxl+WsAQVD1DUP8YgjRvBqDmcuzVIo7EqudIrH82bdrkKExz4xismWx+Wso2qHq2Qf1hC6R5swUaiIjzIuLrEbE3Ii6esP/SiLih+LklIh4Y2XdwZN+uCdd9b0Q8tJbzsPlpuWEAsgmqzjAA2QR1n4uiNU99b4EiYh3wfuDFwBJwfUTsysybhsdk5ptGjn8jcObITfxzZv7IMre9FThmredi89MRNkHVGzZBtkHdZxOkeepxC3Q2sDczb8vMR4CdwPkrHL8duGK1Gy1C1W8Dv7LWE7H56RjXBdXDNqgfbII0L436aoyDj5D77pzXrR0XEXtGLu/IzB3F7ycBd4zsWwLOmXQjEXEKsAW4dmTzEcVtHwDenZmfLLZfBOzKzLsjYk0nafjpMEdi1TME9YOLojUPHR2D3ZeZW5fZNymZ5DLHbgOuysyDI9u+LzPviojvB66NiK8C/wy8EnjhNCfp2KsHHIlVz3FY9zkK07z0aAy2BJw8cnkzsNz/O9/G2MgrM+8q/r0N+HMG64HOBJ4N7I2I24GnRsTe1U7E5qdHHIlVz7fKd5+jMM1Do8Zgi3M9cFpEbAHuZBBwXjV+UEScDmwErhvZthF4ODP3R8RxwI8Dv1Uslj5h5LiHMvPZq52IzU9P2QZVzzao22yCVFbXvyE+Mw8wWJ9zNXAz8NHMvDEiLomIl40cuh3YmZmjI7EfAPZExN8An2Ow5ucmZmTz03OuC6qe64K6zfVA0vIyczewe2zb28cuv3PC9f4/4IfWcPtPW8t5GH4EGILq4EisuxyFSc3W6PCzYcMGX5Qr5rqgetgGdZMhSGqm1qz5cY1K9XzMq+e6oG5yPZDULK0JP0O+IFfPx7x6hqBuMgBJzdDosddKHIdVz5FY9VwX1D2OwqT6tTb8DPmCXA/DZ/VcF9QthiCpPq0be63E8Uz1fMyr50isW1wPJFWv9c3PJLYS1fMxr54jsW7x84Gk6nQy/Aw5Equej3k9HIl1g6MwqRqdGnutxPFM9XzMq+dIrBschUmL1ZvwM+QLcvV8zKtnCOoGQ5C0GJ0ee63ENSrVcyRWPdcFdYPrgaT56m34GfIFuR6Gz+q5LqjdXA+kRclHH+HAPbfXfRqV6t3YayWOZ6rnY149R2Lt5ihMKq/3zc8kthLV8zGvniOxdrMJkmZn+FmBI7Hq+ZjXw5FYe7keSJqeY681cjxTPR/z6jkSaydHYdJ0DD9T8gW5ej7m1TMEtZMhSFqbUuEnIo6JiKsi4m8j4uaI+LGIODYiromIW4t/NxbHRkT8QUTsjYivRMRZ8/kT6uELcvWGj7mPe3WGIcgg1C6GIGllZZuf9wCfzsznAj8M3AxcDHw2M08DPltcBngpcFrxcyHwgZL33Qi+INfDx7x6hqD2MQBJk80cfiLiKOBfA5cBZOYjmfkAcD7woeKwDwEvL34/H/hwDnweOCYiOvXfTF+Qq+djXj1DULvYAkmHKtP8fD/wbeA/R8SXI+KPIuJI4PjMvBug+PeZxfEnAXeMXH+p2PYkEXFhROyJiD3/9E//VOL06uMLcvV8zKvnSKxdDEHSE8qEn/XAWcAHMvNM4Ds8MeKaJCZsy0M2ZO7IzK2ZufXpT396idOrnyOx6vmY18MQ1B6GIKlc+FkCljLzC8XlqxiEoW8Nx1nFv/eOHH/yyPU3A735IBdfkKvnY149Q1B7GIDUZzOHn8y8B7gjIk4vNp0L3ATsAi4otl0AfKr4fRfw88W7vp4PPDgcj/WJL8jV8zGvniGoHWyB1FdlP+H5jcBHIuIw4DbgFxgEqo9GxGuBbwKvLI7dDfwUsBd4uDi2t/w6h+r56dHV8ys02sGvylDflAo/mXkDsHXCrnMnHJvAG8rcXxf5glwPw2f1/AqN5jMEqS/8hOcGcTxTPR/z6jkSaz5HYeo6v9i0gWwlqudjXj1HYs1mC6QuM/w0mCOx6vmY18ORWHMZgtRFhp+WsJmono959QxBzWUI6q58dD8H7rq97tOolGt+WsY1KtXzMa+e64Kay7fHqwtsflrKVqJ6jsSq57qg5tq0aZMtkFrL8NNyviDXw/BZPUdizeMoTG3l2KtDHM9Uz8e8eo7EmsdRmNrG5qeDbCWq52NePUdizWMTpLYw/HSYI7Hq+ZjXw5FYs7geSE3n2KsnHM9Uz8e8eo7EmsNRmJrM8NMzviBXz8e8eoag5jAEqYkce/WUa1Sq50iseq4Lag7XA6lJDD895wtyPQyf1XNdUDO4HkhN0Oix14YNG+o+hV5xPFM9H/PqORKrn6Mw1a3xzc/xxx8PwLe+9a2az6Q/bCWq52NePUdi9XMUpro0PvwMGYKq50isej7m9XAkVi9DkKrW6LHXJMcff/zjP6qO45nq+ZhXz5FYvRyFqSqtCz+jDEHV8wW5ej7m1TME1cf1QN0WEedFxNcjYm9EXDxh/6URcUPxc0tEPDC2/6iIuDMi3jeybXtEfDUivhIRn46I41Y7j9aMvVbiSKx6rlGpniOx6rkuqD6OwronItYB7wdeDCwB10fErsy8aXhMZr5p5Pg3AmeO3cy7gL8YOWY98B7gjMy8LyJ+C7gIeOdK59Lq5mecI7HqDVsJm4lq+ZhXzzaoHjZBnXI2sDczb8vMR4CdwPkrHL8duGJ4ISJ+FDge+MzIMVH8HBkRARwFrPr/DjsVfkYZgqrnC3L1fMyrZwiqhwGoNY6LiD0jPxeO7DsJuGPk8lKx7RARcQqwBbi2uPwU4HeBXx49LjMfBf4d8FUGoecM4LLVTrITY6+VjAYgx2LVcDxTPceQ1XMkVr3RAOQ4bH4ee+RRHvr7O+d1c/dl5tZl9sWEbbnMsduAqzLzYHH59cDuzLxjUPAUNxixgUH4ORO4DXgv8Dbg11c6yc6Hn1HjTZBhaPEMQtXy8a6Hb5WvnmuCWmkJOHnk8maWH1FtA7/IUhIAABcCSURBVN4wcvnHgBdExOuBpwGHRcRDwMcAMvMbABHxUeCQhdTjehV+xtkKVcsX5mrZBlXPEFQ9Q1CrXA+cFhFbgDsZBJxXjR8UEacDG4Hrhtsy8+dG9r8G2JqZF0fEicAZEfGMzPw2g8XUN692Ir0OP6NsharlC3N1fKyrZwiqniOx5svMAxFxEXA1sA64PDNvjIhLgD2Zuas4dDuwMzOXG4mN3uZdEfFrwF9GxKPA3wOvWe16hp9l2ApVwzaoOj7W1XNdUD1sg5orM3cDu8e2vX3s8jtXuY0/Bv545PIHgQ9Ocx6GnzWwFaqGL87VsQ2qnm1Q9QxBWk6jw8/69es59thjAbj//vtrPpsn2AotnkGoGoag6hmCqudITOMaHX5GDUPQUFPCkK3Q4hmEFs/HuHqOxOphGyRoUfgZNxqGmhKEwFZo0XyRXjzboOrZBlXPENRvrQ0/o2yF+skgtFiGoOoZgqpnCOqnToSfcW0IQwah+TIILY6PbfUciVXPdUH90snwM66JIzJbocWxsVgcH9vq2QZVzzao+3oRfkbZCvWHjcXiGIKqZwiqniGouxodftatW8fGjRvZt2/fwu7DVqgfDEKLYQiqniOx6jkS655Gh5+hjRs3PunyosKQrVA/GITmz8e0HrZB1bMN6oZWhJ9xo2GoqlYImhGGbIXmyxft+bMNqp4hqHqGoHZrZfgZVVUrBM0fkRmEyjEIzZchqHqOxKrnSKydWh9+xtkKPcEwNDuD0Pz4WNbDNqh6tkHt0bnwM8pWyFZoHnzxnh/boOoZgqrXthB0cP8jPPiNO+s+jUp1OvyMsxUaMAjNzhfv+fBxrJ4hqHqOxJqr0eFn3bp1C7vtulohqD8MOR4rzzZoPnwcq+e6oHq0rQ3qukaHH4Cjjz768d8ffPDBhd1Pn0dktkLl+AI+H7ZB1bMNqp4hqBkaH35GjQYhqC4M2QpprQxC5RmCqmcIqp4jsXqVDj8RsQ7YA9yZmT8dEVuAncCxwJeAV2fmIxFxOPBh4EeBfwB+NjNvL3PftkKLZys0O4NQOT5+1XMkVg/boOrNo/n598DNwFHF5d8ELs3MnRHxQeC1wAeKf/dl5rMjYltx3M/O4f4BW6Eq2ArNzhfycmyDqmcbVL3RNkiLVSr8RMRm4N8CvwG8OSICeBHwquKQDwHvZBB+zi9+B7gKeF9ERGZmmXNYTlVhyIXTAwahtTMIzc4QVD1DkLqobPPz+8CvAE8vLn8v8EBmHiguLwEnFb+fBNwBkJkHIuLB4vj7Rm8wIi4ELgQ4+eSTS57eExyRLZat0Gx8MZ+NAbJ6jsTUJTOHn4j4aeDezPxiRLxwuHnCobmGfU9syNwB7AA488wzW90KgSMyMAithS/mszNAVs82SG1Xpvn5ceBlEfFTwBEM1vz8PnBMRKwv2p/NwPB/kZaAk4GliFgPHA2s+Aq9bt06jjpqsJToH//xH0uc6spshRbLVmg6BqHZGIKqZwhSW80cfjLzbcDbAIrm5y2Z+XMR8SfAKxi84+sC4FPFVXYVl68r9l87zXqfYQiC6oIQ2Aotgq3Q2hmEpmcIqp4jMbXNIj7n563Azoj4deDLwGXF9suA/xIRexk0PttmvYPRIAS2QtNqUhiyFVo7g9B0fLzqYRukNphL+MnMPwf+vPj9NuDsCcd8F3jlPO5vXFVhqK5WCPo5IjMILc8X9unYBlXPEKQma9UnPK+VI7LZ2Qq1j0Fo7QxB1XMkpibqZPgZ5YisHFuhdjEIrY2PUz1sg9QUnQ8/42yFZmcr1C62HGvj41Q9Q5Dq1uzwc+DRhd68rVA5TQ1DBqEns+VYG0NQ9QxBqkuzww/AvrH/Idp44uTj5sCF0+U0ZURmK7Q8g9DqfIyq57qgeh3Yf5B9f1f/F2lXqfnhZ1xNYcgR2XRshZrPF/nV2QZVzzZIVWhf+Bk3GoY60ApB90dkTQlCYBgaMgitzBBUPUOQFqn94WeUrVAptkIGITAIrcTHpnqOxLQI3Qo/42yFZtbHhdO2QofyxX55tkHVsw3SvHQ7/Ixy4XQpfR+RGYQMQssxBFXPEKSy+hN+xjkiK6VvIzJboSfzBf9QhsPqORLTrJodfg4+Qu678/GLsfGkxd2XI7KZ2Qr1Nwj5gj+Z4bB6tkGaRrPDz5haghDYCk3JVqifYcggdChDUPUMQVqLVoWfUaNBCGyFpmUrtDi2QgahcYag6jkS00paG37G2QrNrosLp22FmsMg9AQfi3rYBmlcZ8LPqNpaIVhYGKqrFQJHZPPU91bIF/8n2AZVzxCkoU6Gn3GOyMpxRLYYBiGDEBiC6uBITL0IP+Mckc3OVmgx+j4eMwj5GNTFNqifGh1+8tFHOHDP7aw/4dTF3YetUCm2QovR51bIJsTHoA6GoGpExHnAe4B1wB9l5rvH9l8K/ERx8anAMzPzmJH9RwE3A5/IzIsi4qnAnwDPAg4C/zUzL17tPBodfoYO3HP7ky5XFYZshabjwunF6GsrZBNiCKqDIWhxImId8H7gxcAScH1E7MrMm4bHZOabRo5/I3Dm2M28C/iLsW2/k5mfi4jDgM9GxEsz889WOpdWhJ9xo2Gok60QuHB6Sn0dkRmE+qHvf38dXBe0EGcDezPzNoCI2AmcD9y0zPHbgXcML0TEjwLHA58GtgJk5sPA54rfH4mILwGbD72pJ2tl+BlVVysEjsim5Yhs/vrYCvU9CNgGVc82aCrHRcSekcs7MnNH8ftJwB0j+5aAcybdSEScAmwBri0uPwX4XeDVwLnLXOcY4H9hMFZbUevDzzhHZOV0fURmK9QtfQ5ChqDqdTUEHdx/gH23PTCvm7svM7cusy8mbMtljt0GXJWZB4vLrwd2Z+YdEYfeTESsB64A/mDYLK2kc+FnXOdHZLZCU7EV6q6+BqG+/t11ciQ2syXg5JHLm4Hl/kO7DXjDyOUfA14QEa8HngYcFhEPjSxu3gHcmpm/v5YT6Xz4GWUrVI6t0OxsharV10BgG1S9rrZBC3I9cFpEbAHuZBBwXjV+UEScDmwErhtuy8yfG9n/GmDrMPhExK8DRwOvW+uJNDr85KP7F3r7tkKz6+LC6bpaIag+DPWpFepjIOjj31w3Q9DqMvNARFwEXM3gre6XZ+aNEXEJsCczdxWHbgd2ZuZyI7HHRcRm4FeBvwW+VIzE3peZf7Ti9dZw27U561mb86/e/UuPX15/4qmV3fciw9C4hYahUQsMQ+MWGYZGLbIVGrfIMDSqzi9lhW4HoaG+hYK+/b1NMWsQuvLKK7+4wrqZuXvOEU/N937fc+dyW+fd+uVKz31WjW5+xh246/YnXV5kGHJEVo4jstnZCi1e38Zifft7m8I2qLlaFX7G1RWGHJFNx4XT5TRp4bRBqP0ciVXPENQ8rQ4/40bDkK3QlGyFSrEV6oY+BSFDUPV8l1hzdCr8jLIVKqnjrRC4cHoeutwK9SUI9eXvbBrboHp1NvyM63orBAsMQzW1QuCIbFp1jsi63Ar1JSDYBlXPEFSP3oSfUS6cLskRWSl9HJEZhNrFEFS90ZGYFq/R4eexRx7lob8fvKA/7ZTFvZg7Iiup4yMyW6HyutoKdT0k9CHoqZ8aHX5GDUPQUFVhyFZoSrZCpdgKtVMfQkLXg576pTXhZ9xoGLIVmo6tUDldb4Wg2jDUtVao60HIEKQuaG34GWUrVE7XWyFYXBiqqxWCfozIDELN1eW/Td3XifAzruutECwuDNXWCoEjsin1YUTWpVaoy2HBNkht08nwM6quVggckU3NEdnMbIXapatByBCktuh8+BnniKycro/IbIWmYytUXheDUBf/JnVLo8PPwf2P8OA37uToZy3uRbbrIzJboenYCpXTlDBkEGoO26Dme+TAYyzd93Ddp1GpRoefoQe/8cSLbFVBCGyFptX1VghcOD2tukZkXWiFuhYauvb3qN1aEX5GjQYhqC4MdbEVAhdOT8sR2exshWbTtTbIEKQmaF34GVdVGHLhdDmOyMrp+ojMVmhtuhSEuvS3qH1aH37GOSIrcT+OyEqxFZqdrdD0uhQebINUtc6Fn1GOyErel63QzGyFyqkrDBmE6mUIUlVmDj8RcTLwYeAE4DFgR2a+JyKOBa4ETgVuB34mM/dFRADvAX4KeBh4TWZ+qdzpT8dWqMT92AqV4sLpcuoYkbV1PNaFINSFv0HNVqb5OQD8X5n5pYh4OvDFiLgGeA3w2cx8d0RcDFwMvBV4KXBa8XMO8IHi31rYCpW8r663QuCIbEpdH5G1sRXqQoiwDdIizBx+MvNu4O7i93+KiJuBk4DzgRcWh30I+HMG4ed84MOZmcDnI+KYiNhU3M5EB/YfZN/fPfE/bBu3HLvcoaV1vRWCxYWhulohcEQ2ra6PyGyFltf2ENH281ezzGXNT0ScCpwJfAE4fhhoMvPuiHhmcdhJwB0jV1sqtj0p/ETEhcCFACcccfiT7mc0CMHiwlBdrRA4IptW10dktkLTsRVaXdvboLafv5qhdPiJiKcBHwP+Q2b+42Bpz+RDJ2zLQzZk7gB2APzA0Ucdsn9UHa0QOCKb+r66PiKzFZqKrVBztD1I2AZpVqXCT0RsYBB8PpKZHy82f2s4zoqITcC9xfYl4OSRq28G5vaf2KpaIej+iMxWaEq2QqVUFYZshVbW5iBkCNK0yrzbK4DLgJsz8/dGdu0CLgDeXfz7qZHtF0XETgYLnR9cab1PWbZCs7MVKqmiMFRXKwSOyGbVllaorUGoreet6pVpfn4ceDXw1Yi4odj2HxmEno9GxGuBbwKvLPbtZvA2970M3ur+CyXueyq2QuW4cLokR2Qz69OIzCA0X7ZBWkmZd3v9FZPX8QCcO+H4BN4w6/3Nkwuny3FEVoIjslJsherVxiBkCNIkjf6E54P7D7DvtgcA2Pj9xyzsfhyRzc4RWUm2QjOzFapX24JQ285Xi9Xo8DNqGIKguiAEjsimZStUgq1QKV1eON30Vqht7Urbzlfz15rwM2o0CIGt0LRshWbnwulyHJHNR1Nboba1K4ag/mpl+BlnKzQ7F06X44isHEdk5TW1FWpTEOp7CNr/WPKN7zxa92lUqhPhZ5StUDmOyMpxRDY7W6H5aGIr1JYg1JbzVHmdCz/jqgpDdbVC4Ihs6vvq+ojMVmgqtkLVakvA6Hsb1HWdDz/jHJHNzlaoHFuh2dkKzUfTWqE2BCFDUDc1Ovw8cuCxhd6+I7JybIVmZytUTlVhqK5WCBYfhprWCjU9CDX9/DSdRocfgKX7Hn7S5c3HPXVh92UrNDtboXIqC0M1tULgiGxadY7IDEIrsw1qv8aHn3GjYaiqIAS2QtOqKgzV1QqBI7JpOSKbXZ9boSYHjSafm1bWuvAzqq5WCFw4PS1HZLNzRFaOrVB5TWiFmtwGNfncNFmrw884R2TlOCIrcT+2QqXYCs2uj61Qk8OGbVA7dCr8jHNENjtboZL3ZSs0MxdOl1NnGDIIPcEQ1GydDj+jbIXKsRUqcT8unC7FEVk5VY7I6m6FmhiEmnhO6lH4GWcrNDsXTpe8L0dkpTgim12fWqEmhg7boOZodPgZ/b6RZx25YWH348LpchyRlbwvR2QzsxUqpy+tUNOCkCGofo0OP6PGv3Sti2HIEdl0HJGVYys0O1uh8upqhZoUhJp0Ln3TmvAzrq4w1IVWCLo/IrMVmo6tUDldb4VgsWGorlaoSQ1Mk86lD1obfsaNhiFboSnvx1aoFFuhEjq4cLquVgi6OSKrIgg1qYEZPZcuiojzgPcA64A/ysx3j+2/FPiJ4uJTgWdm5jERcQrw8eJ6G4D3ZuYHi+scBrwPeCHwGPCrmfmxlc6jM+FnlK1QyfuyFZpZFxdO19YKgSOyKXVxRNbnINQ1EbEOeD/wYmAJuD4idmXmTcNjMvNNI8e/ETizuHg38K8yc39EPA34WnHdu4BfBe7NzOdExFOAVV+4Ohl+xtkKlbifDi6crqsVAkdkU3NENrMutkJVj8cMQnN3NrA3M28DiIidwPnATcscvx14B0BmPjKy/XDgKSOXfxF4bnHcY8B9q51IZOa0J1+ZiPg28Pd1n8ccHMcangxVwueiOXwumsXnozlOz8ynV3VnEfFpBs//PBwBfHfk8o7M3FHczyuA8zLzdcXlVwPnZOZFE87pFODzwObMPFhsOxn4U+DZwC9n5vsj4hjgq8CfMBh7fQO4KDNXTMaNbn4y8xl1n8M8RMSezNxa93nI56JJfC6axeejOSJiT5X3l5nnVXRXMenulzl2G3DVMPgAZOYdwPMi4kTgkxFxFXAQ2Az8dWa+OSLeDPwO8OqVTuQpK+2UJEmakyXg5JHLm4Hl5onbgCsm7SjW+dwIvAD4B+Bh4BPF7j8BzlrtRAw/kiSpCtcDp0XEluIdWtuAXeMHRcTpwEbgupFtmyPie4rfNwI/Dnw9B2t3/iuDkRfAuSy/huhxjR57dciOuk9Aj/O5aA6fi2bx+WiOTj4XmXkgIi4CrmbwlvXLM/PGiLgE2JOZwyC0HdiZT16U/APA70ZEMhif/U5mfrXY91bgv0TE7wPfBn5htXNp9IJnSZKkeXPsJUmSesXwI0mSesXwU1JEnBcRX4+IvRFx8YT9h0fElcX+L0TEqcX2F0fEFyPiq8W/L6r63Ltm1udiZP/3RcRDEfGWqs65q8o8FxHxvIi4LiJuLP77cUSV5941Jf43akNEfKh4Dm6OiLdVfe5ds4bn4l9HxJci4kDxmTij+y6IiFuLnwuqO+uOykx/ZvxhsGDrG8D3A4cBfwOcMXbM64EPFr9vA64sfj8TOLH4/QeBO+v+e9r8U+a5GNn/MQZvk3xL3X9Pm39K/vdiPfAV4IeLy98LrKv7b2rrT8nn4lUMFp3C4DuWbgdOrftvauvPGp+LU4HnAR8GXjGy/VjgtuLfjcXvG+v+m9r8Y/NTzuMf1Z2Dj94eflT3qPOBDxW/XwWcGxGRmV/OwWcVwODzCo6IiMMrOetumvm5AIiIlzP4H5QbKzrfLivzXLwE+Epm/g1AZv5DjnzImaZW5rlI4MiIWA98D/AIsLjv/ei+VZ+LzLw9M7/C4Ms5R/0kcE1m3p+Z+4BrgKo+mLCTDD/lnATcMXJ5qdg28ZjMPAA8yOD/zY7634EvZ+b+BZ1nH8z8XETEkQzeKvlrFZxnH5T578VzgIyIq4v6/1cqON8uK/NcXAV8h8EXSn6TwVuLF/t17t22ludiEdfVBH7OTzlr+ajuFY+JiH8B/CaD/8er2ZV5Ln4NuDQzHyqKIJVT5rlYD/xPwL9k8Kmtn42IL2bmZ+d7ir1R5rk4m8FXB5zIYNTy3yPiv2XxpZSa2jRf7TDP62oCm59y1vJR3Y8fU9THRwP3F5c3M/hI7p/PzG8s/Gy7rcxzcQ7wWxFxO/AfgP9YfBCXZlPmuVgC/iIz78vMh4HdrOGj6rWsMs/Fq4BPZ+ajmXkv8NeA3/01u2m+2mGe19UEhp9y1vJR3buA4cr8VwDXZmYW30T7p8DbMvOvKzvj7pr5ucjMF2TmqZl5KvD7wP+Tme+r6sQ7aObngsEnvz4vIp5avBD/G9bwUfVaVpnn4pvAi2LgSOD5wN9WdN5dtKavdljG1cBLImJj8dUOLym2aUaGnxKK+fjwo7pvBj6axUd1R8TLisMuY7CuZC/wZmD49saLgGcD/3dE3FD8PLPiP6EzSj4XmqMyz0WxmPP3GLxQ3AB8KTP/tOq/oStK/vfi/cDTgK8xeD7+c7EYVzNYy3MREf8yIpaAVwL/b0TcWFz3fuBdDJ6H64FLXH9Vjl9vIUmSesXmR5Ik9YrhR5Ik9YrhR5Ik9YrhR5Ik9YrhR5Ik9YrhR5Ik9YrhR5Ik9cr/D5VfTNK35hi/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.contourf(LR, NE, Z2, 20, cmap='RdGy')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "1/(np.var(X_train)*X_train.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best params \n",
    "## DeepDense VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.load(\"data/encoded_data_train_DeepDenseVAE_mark_V_[160-256-128-15]_VDO_GNLL_400ep_lr-1e-4.npy\")\n",
    "y_train = np.load(\"data/labels_train.npy\")\n",
    "X_test = np.load(\"data/encoded_data_test_DeepDenseVAE_mark_V_[160-256-128-15]_VDO_GNLL_400ep_lr-1e-4.npy\")\n",
    "y_test = np.load(\"data/labels_test.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd_clf = GradientBoostingClassifier(n_estimators=500, learning_rate=0.1, subsample=0.7, n_iter_no_change=50, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Iter       Train Loss      OOB Improve   Remaining Time \n",
      "         1       36131.2409        1960.4802            4.81m\n",
      "         2       33622.5183         991.0259            4.68m\n",
      "         3       31907.7924         764.3794            5.19m\n",
      "         4       30344.8810         598.4269            5.05m\n",
      "         5       29238.9626         522.5916            5.18m\n",
      "         6       28199.3410         413.4237            5.21m\n",
      "         7       27495.1159         325.4065            5.13m\n",
      "         8       26708.6730         323.8698            5.12m\n",
      "         9       26146.1703         256.2357            5.42m\n",
      "        10       25422.6371         252.1090            5.76m\n",
      "        20       22180.7544          92.5340            5.28m\n",
      "        30       20562.5346          59.1883            5.09m\n",
      "        40       19456.6186          26.2112            5.10m\n",
      "        50       18711.4918          26.9687            4.99m\n",
      "        60       18213.8957          12.8264            5.01m\n",
      "        70       17683.1057          13.6030            4.91m\n",
      "        80       17292.5532           5.0260            4.73m\n",
      "        90       17024.7462           3.6201            4.66m\n",
      "       100       16742.6072           2.0481            4.44m\n",
      "       200       14844.4828           0.2255            2.97m\n",
      "       300       13850.0742          -0.1816            1.88m\n",
      "       400       13209.6513          -0.2875           54.84s\n",
      "       500       12499.2337          -0.6713            0.00s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "                           learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "                           max_features=None, max_leaf_nodes=None,\n",
       "                           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                           min_samples_leaf=1, min_samples_split=2,\n",
       "                           min_weight_fraction_leaf=0.0, n_estimators=500,\n",
       "                           n_iter_no_change=50, presort='auto',\n",
       "                           random_state=None, subsample=0.7, tol=0.0001,\n",
       "                           validation_fraction=0.1, verbose=True,\n",
       "                           warm_start=False)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_clf.n_estimators_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models_and_losses/Gradient_Tree_Boosting_DeepDenseVAE{n_estim=500-early_stopping,lr=0,1,subsample=0,7}.joblib']"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#joblib.dump(dd_clf, \"models_and_losses/Gradient_Tree_Boosting_DeepDenseVAE{n_estim=500-early_stopping,lr=0,1,subsample=0,7}.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_dd = dd_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8167759394182772"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred = y_pred_dd, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.857373326959847"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred = y_pred_dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2884, 1185,   26,  140],\n",
       "       [ 405, 9874,   34,  132],\n",
       "       [  25,   48,  391,   15],\n",
       "       [ 185,  179,   13, 1200]])"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred = y_pred_dd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN-VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.load(\"data/encoded_data_train_DeepConvVAE_ELU_NLL_250ep_lr-{1e-3, 1e-4}_bs-300.npy\")\n",
    "X_test = np.load(\"data/encoded_data_test_DeepConvVAE_ELU_NLL_250ep_lr-{1e-3, 1e-4}_bs-300.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "dc_clf = GradientBoostingClassifier(n_estimators=1000, learning_rate=0.1, subsample=0.7, n_iter_no_change=50, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Iter       Train Loss      OOB Improve   Remaining Time \n",
      "         1       35928.6610        1994.3430           10.00m\n",
      "         2       33612.0603        1029.3719            9.42m\n",
      "         3       31690.2714         775.5264            9.20m\n",
      "         4       30349.4240         610.3557            9.12m\n",
      "         5       29206.7606         492.8922            9.28m\n",
      "         6       28169.1437         406.5100            9.26m\n",
      "         7       27298.3573         352.7039            9.35m\n",
      "         8       26622.9455         291.5632            9.58m\n",
      "         9       25921.8196         269.1957            9.72m\n",
      "        10       25512.4242         222.6032            9.62m\n",
      "        20       21998.7155          91.1914            9.04m\n",
      "        30       20396.1107          28.4340            8.85m\n",
      "        40       19425.7226          19.7256            8.65m\n",
      "        50       18830.8415          12.0093            8.53m\n",
      "        60       18199.8996          16.6281            8.59m\n",
      "        70       17755.0351          10.3283            8.70m\n",
      "        80       17441.8054           9.7543            8.52m\n",
      "        90       17093.6050           8.9928            8.34m\n",
      "       100       16905.2554           7.7986            8.20m\n",
      "       200       15054.0220           0.8772            7.01m\n",
      "       300       14112.8184          -0.0210            6.03m\n",
      "       400       13385.3481          -0.3559            5.20m\n",
      "       500       12796.3853           0.8355            4.29m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "                           learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "                           max_features=None, max_leaf_nodes=None,\n",
       "                           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                           min_samples_leaf=1, min_samples_split=2,\n",
       "                           min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                           n_iter_no_change=50, presort='auto',\n",
       "                           random_state=None, subsample=0.7, tol=0.0001,\n",
       "                           validation_fraction=0.1, verbose=True,\n",
       "                           warm_start=False)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "579"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc_clf.n_estimators_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models_and_losses/Gradient_Tree_Boosting_CNN-VAE{n_estim=579-early_stopping,lr=0,1,subsample=0,7}.joblib']"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#joblib.dump(dc_clf, \"models_and_losses/Gradient_Tree_Boosting_CNN-VAE{n_estim=579-early_stopping,lr=0,1,subsample=0,7}.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_dc = dc_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8163829270773644"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred = y_pred_dc, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8560587954110899"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred = y_pred_dc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2851, 1227,   23,  134],\n",
       "       [ 436, 9860,   42,  107],\n",
       "       [  26,   46,  391,   16],\n",
       "       [ 172,  160,   20, 1225]])"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred = y_pred_dc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM-VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train1 = np.load(\"data/mu_lstm1.npy\")\n",
    "X_train2 = np.load(\"data/mu_lstm2.npy\")\n",
    "X_train3 = np.load(\"data/mu_lstm3.npy\")\n",
    "\n",
    "X_train = np.vstack((X_train1, X_train2, X_train3))\n",
    "\n",
    "X_test = np.load(\"data/mu_lstm_test.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "dr_clf = GradientBoostingClassifier(n_estimators=1000, learning_rate=0.1, subsample=0.7, n_iter_no_change=50, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Iter       Train Loss      OOB Improve   Remaining Time \n",
      "         1       35513.6796        2241.4136            9.42m\n",
      "         2       32892.3387        1094.0257            9.41m\n",
      "         3       31146.9739         800.8722            9.14m\n",
      "         4       29506.8873         616.6117            9.28m\n",
      "         5       28277.0804         508.9576            9.15m\n",
      "         6       27309.3442         394.3256            9.85m\n",
      "         7       26650.6110         331.9223            9.77m\n",
      "         8       25981.6419         273.4865            9.66m\n",
      "         9       25328.5222         282.8019            9.61m\n",
      "        10       24848.7343         227.3767            9.50m\n",
      "        20       21507.1352          71.6896            9.40m\n",
      "        30       19880.8460          43.1108           10.07m\n",
      "        40       19093.0261          23.5123            9.65m\n",
      "        50       18501.6901          11.8172            9.38m\n",
      "        60       18071.4008           5.8146            9.04m\n",
      "        70       17764.7936           8.4189            8.83m\n",
      "        80       17613.9463           7.9597            8.60m\n",
      "        90       17422.5205           3.2065            8.41m\n",
      "       100       17199.1549           7.3237            8.23m\n",
      "       200       16023.1962          -0.0290            6.99m\n",
      "       300       15191.8447          -1.6179            6.00m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "                           learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "                           max_features=None, max_leaf_nodes=None,\n",
       "                           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                           min_samples_leaf=1, min_samples_split=2,\n",
       "                           min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                           n_iter_no_change=50, presort='auto',\n",
       "                           random_state=None, subsample=0.7, tol=0.0001,\n",
       "                           validation_fraction=0.1, verbose=True,\n",
       "                           warm_start=False)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "362"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr_clf.n_estimators_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models_and_losses/Gradient_Tree_Boosting_LSTM-VAE{n_estim=368-early_stopping,lr=0,1,subsample=0,7}.joblib']"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#joblib.dump(dc_clf, \"models_and_losses/Gradient_Tree_Boosting_LSTM-VAE{n_estim=362-early_stopping,lr=0,1,subsample=0,7}.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_dr = dr_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.792792574984842"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred = y_pred_dr, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.831560707456979"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred = y_pred_dr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2491, 1577,   32,  135],\n",
       "       [ 459, 9795,   46,  145],\n",
       "       [  20,   43,  399,   17],\n",
       "       [ 162,  156,   27, 1232]])"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred = y_pred_dr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}