submit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "import tqdm\n",
    "import re\n",
    "import time\n",
    "from string import punctuation\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch, torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "sns.set_style('darkgrid')\n",
    "mpl.rcParams['figure.figsize'] = [15,10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kaggle.competitions import nflrush"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "import datetime\n",
    "import re\n",
    "\n",
    "def crps(y_true, y_pred):\n",
    "    return np.mean(np.square(y_true - y_pred), axis=1)\n",
    "\n",
    "def yard_to_cdf(yard):\n",
    "    yard = np.round(yard).astype(int)\n",
    "    indices = yard+99\n",
    "    cdfs = np.zeros((yard.shape[0], 199))\n",
    "    for i in range(len(cdfs)):\n",
    "        cdfs[i, indices[i]:] = 1\n",
    "    return cdfs\n",
    "\n",
    "def cdf_to_yard(cdf):\n",
    "    yard_index = (cdf==1).argmax(axis=1)\n",
    "    yard = yard_index-99\n",
    "    return yard\n",
    "\n",
    "\n",
    "def cdf_to_yard_torch(cdf):\n",
    "    yard_index = torch.sum((torch.as_tensor(cdf) <= 0),dim=1)\n",
    "    yard = yard_index-99\n",
    "    return yard\n",
    "\n",
    "def crps_torch(y_true, y_pred):\n",
    "    y_true = torch.as_tensor(y_true)\n",
    "    y_pred = torch.as_tensor(y_pred)\n",
    "    return torch.mean((y_true - y_pred).pow(2), dim=1)\n",
    "\n",
    "def crps_loss(y_true, y_pred_pdf):\n",
    "    y_pred_cdf = torch.cumsum(torch.as_tensor(y_pred_pdf), dim=1)\n",
    "    return crps_torch(y_true, y_pred_cdf).mean()\n",
    "\n",
    "def crps_loss_cdf(y_true, y_pred_cdf):\n",
    "    return crps_torch(y_true, y_pred_cdf).mean()\n",
    "    \n",
    "def clean_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    txt = txt.lower()\n",
    "    txt = ''.join([c for c in txt if c not in punctuation])\n",
    "    txt = re.sub(' +', ' ', txt)\n",
    "    txt = txt.strip()\n",
    "    txt = txt.replace('outside', 'outdoor')\n",
    "    txt = txt.replace('outdor', 'outdoor')\n",
    "    txt = txt.replace('outddors', 'outdoor')\n",
    "    txt = txt.replace('outdoors', 'outdoor')\n",
    "    txt = txt.replace('oudoor', 'outdoor')\n",
    "    txt = txt.replace('indoors', 'indoor')\n",
    "    txt = txt.replace('ourdoor', 'outdoor')\n",
    "    txt = txt.replace('retractable', 'rtr.')\n",
    "    return txt\n",
    "\n",
    "def transform_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    if 'outdoor' in txt or 'open' in txt:\n",
    "        return 1\n",
    "    if 'indoor' in txt or 'closed' in txt:\n",
    "        return 0\n",
    "    return np.nan\n",
    "\n",
    "def str_to_seconds(txt):\n",
    "    txt = txt.split(':')\n",
    "    ans = int(txt[0])*60 + int(txt[1]) + int(txt[2])/60\n",
    "    return ans\n",
    "\n",
    "def str_to_float(txt):\n",
    "    try:\n",
    "        return float(txt)\n",
    "    except Exception as e:\n",
    "        return np.NaN\n",
    "\n",
    "def map_weather(txt):\n",
    "    ans = 1\n",
    "    if pd.isna(txt):\n",
    "        return 0\n",
    "    if 'partly' in txt:\n",
    "        ans*=0.5\n",
    "    if 'climate controlled' in txt or 'indoor' in txt:\n",
    "        return ans*3\n",
    "    if 'sunny' in txt or 'sun' in txt:\n",
    "        return ans*2\n",
    "    if 'clear' in txt:\n",
    "        return ans\n",
    "    if 'cloudy' in txt:\n",
    "        return -ans\n",
    "    if 'rain' in txt or 'rainy' in txt:\n",
    "        return -2*ans\n",
    "    if 'snow' in txt:\n",
    "        return -3*ans\n",
    "    return 0\n",
    "\n",
    "def new_orientation(angle, play_direction):\n",
    "    if play_direction == 0:\n",
    "        new_angle = 360.0 - angle\n",
    "        if new_angle == 360.0:\n",
    "            new_angle = 0.0\n",
    "        return new_angle\n",
    "    else:\n",
    "        return angle\n",
    "\n",
    "def preprocess_features(df, fillna=False):\n",
    "    \"\"\"Accepts df like train data, returns X, y\"\"\"\n",
    "\n",
    "    # Feature engineering\n",
    "    df['DefendersInTheBox_vs_Distance'] = (df['DefendersInTheBox'] / df['Distance'])\n",
    "    df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)\n",
    "    df['StadiumTypeShort'] = df['StadiumType'].apply(transform_StadiumType)\n",
    "    df = df.drop(['StadiumType'], axis=1)\n",
    "\n",
    "    Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', \n",
    "        'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', \n",
    "        'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', \n",
    "        'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', \n",
    "        'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'} \n",
    "\n",
    "    turf_type = df['Turf'].map(Turf)\n",
    "    df['TurfIsNatural'] = (turf_type == 'Natural')\n",
    "    df = df.drop(['Turf'], axis=1)\n",
    "\n",
    "    # CAREFUL. What if a new team appears?\n",
    "    map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}\n",
    "    for abb in df['PossessionTeam'].unique():\n",
    "        map_abbr[abb] = abb\n",
    "\n",
    "    def safe_map(val):\n",
    "        if map_abbr.get('val'):\n",
    "            return map_abbr[val]\n",
    "        else:\n",
    "            return val\n",
    "\n",
    "    df['PossessionTeam'] = df['PossessionTeam'].apply(safe_map)\n",
    "    df['HomeTeamAbbr'] = df['HomeTeamAbbr'].apply(safe_map)\n",
    "    df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].apply(safe_map)\n",
    "\n",
    "    df['HomePossesion'] = (df['PossessionTeam'] == df['HomeTeamAbbr'])\n",
    "\n",
    "    df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']\n",
    "    df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']\n",
    "\n",
    "    # in posession\n",
    "    df['InPosession']=(((df.Team == 'home') & (df.PossessionTeam == df.HomeTeamAbbr)) | ((df.Team == 'away') & (df.PossessionTeam == df.VisitorTeamAbbr)))\n",
    "\n",
    "    # Formation columns\n",
    "    df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='OffenseFormation')], axis=1)\n",
    "    # Filling missing dummy columns at test stage\n",
    "    expected_columns = ['OffenseFormation_ACE',\n",
    "         'OffenseFormation_EMPTY',\n",
    "         'OffenseFormation_JUMBO',\n",
    "         'OffenseFormation_PISTOL',\n",
    "         'OffenseFormation_SHOTGUN',\n",
    "         'OffenseFormation_SINGLEBACK',\n",
    "         'OffenseFormation_WILDCAT',\n",
    "         'OffenseFormation_I_FORM']\n",
    "    for col in expected_columns:\n",
    "        if not col in df.columns:\n",
    "            df[col] = 0\n",
    "\n",
    "\n",
    "    df['GameClock'] = df['GameClock'].apply(str_to_seconds)\n",
    "\n",
    "    df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))\n",
    "    df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)\n",
    "\n",
    "    df['TimeHandoff'] = pd.to_datetime(df['TimeHandoff'], utc=True)\n",
    "    df['TimeSnap'] = pd.to_datetime(df['TimeSnap'], utc=True)\n",
    "    df['TimeDelta'] = (df['TimeHandoff']-df['TimeSnap']).apply(lambda x: x.total_seconds())\n",
    "    df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n",
    "    df['PlayerBirthDate'] = pd.to_datetime(df['PlayerBirthDate'], utc=True)\n",
    "\n",
    "    seconds_in_year = 60*60*24*365.25\n",
    "    df['PlayerAge'] = (df['TimeHandoff']-df['PlayerBirthDate']).apply(lambda x: x.total_seconds())/seconds_in_year\n",
    "    df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1)\n",
    "\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: (int(x.split('-')[0])+int(x.split('-')[1]))/2 if not pd.isna(x) and '-' in x else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: (int(x.split()[0])+int(x.split()[-1]))/2 if not pd.isna(x) and type(x)!=float and 'gusts up to' in x else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)\n",
    "\n",
    "    df = df.drop(['WindDirection'], axis=1)\n",
    "    df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')\n",
    "    df['IsHomeTeam'] = df['Team'].apply(lambda x: x.strip()=='home')\n",
    "\n",
    "\n",
    "\n",
    "    df['GameWeather'] = df['GameWeather'].str.lower()\n",
    "    indoor = \"indoor\"\n",
    "    df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x) and indoor in x else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy').replace('clouidy', 'cloudy').replace('party', 'partly') if not pd.isna(x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('skies', '').replace(\"mostly\", \"\").strip() if not pd.isna(x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(map_weather)\n",
    "\n",
    "    df['IsRusher'] = df['NflId'] == df['NflIdRusher']\n",
    "    df.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True)\n",
    "\n",
    "    df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)\n",
    "    df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)\n",
    "    df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)\n",
    "    \n",
    "    df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)\n",
    "    df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)\n",
    "    \n",
    "\n",
    "    # DefensePersonnel\n",
    "    counts = []\n",
    "    for i, val in df['DefensePersonnel'].str.split(',').iteritems():\n",
    "        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0, 'QB': 0}\n",
    "        if val is np.NaN:\n",
    "            counts.append({})\n",
    "            continue\n",
    "        for item in val:\n",
    "            name, number = item.strip().split(' ')[::-1]\n",
    "            row[name] = int(number)\n",
    "        counts.append(row)\n",
    "    defense_presonell_df = pd.DataFrame(counts)\n",
    "    defense_presonell_df.columns = ['defense_'+x for x in defense_presonell_df.columns]\n",
    "    defense_presonell_df = defense_presonell_df.fillna(0).astype(int)\n",
    "    defense_presonell_df.index = df.index\n",
    "    df = pd.concat([df.drop(['DefensePersonnel'], axis=1), defense_presonell_df], axis=1)\n",
    "\n",
    "\n",
    "    # OffensePersonnel\n",
    "    counts = []\n",
    "    for i, val in df['OffensePersonnel'].str.split(',').iteritems():\n",
    "        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0, 'QB': 0}\n",
    "        if val is np.NaN:\n",
    "            counts.append({})\n",
    "            continue\n",
    "        for item in val:\n",
    "            name, number = item.strip().split(' ')[::-1]\n",
    "            row[name] = int(number)\n",
    "        counts.append(row)\n",
    "    offense_personnel_df = pd.DataFrame(counts)\n",
    "    offense_personnel_df.columns = ['offense_'+x for x in offense_personnel_df.columns]\n",
    "    offense_personnel_df = offense_personnel_df.fillna(0).astype(int)\n",
    "    offense_personnel_df.index = df.index\n",
    "    df = pd.concat([df.drop(['OffensePersonnel'], axis=1), offense_personnel_df], axis=1)\n",
    "    \n",
    "\n",
    "    df = sort_df(df)\n",
    "\n",
    "    if fillna:\n",
    "        df.fillna(-999, inplace=True)\n",
    "    return df\n",
    "\n",
    "def sort_df(df):\n",
    "    df = df.sort_values(by=['PlayId', 'InPosession', 'IsRusher']).reset_index(drop=True)\n",
    "    return df\n",
    "\n",
    "def make_x(df):\n",
    "    source_play_id = df['PlayId']\n",
    "\n",
    "    cols_delete = ['GameId', 'PlayId', 'IsRusher', 'Team']\n",
    "    df = df.drop(cols_delete, axis=1)\n",
    "\n",
    "    # Fill nan\n",
    "\n",
    "    # Text features\n",
    "    text_cols = [\n",
    "        'FieldPosition',\n",
    "         'DisplayName',\n",
    "         'PossessionTeam',\n",
    "         'PlayerCollegeName',\n",
    "         'Position',\n",
    "         'HomeTeamAbbr',\n",
    "         'VisitorTeamAbbr',\n",
    "         'Stadium',\n",
    "         'Location']\n",
    "    df = df.drop(text_cols, axis=1)\n",
    "\n",
    "    # Player features\n",
    "    cols_player = ['X',\n",
    "         'Y',\n",
    "         'S',\n",
    "         'A',\n",
    "         'Dis',\n",
    "         'Orientation',\n",
    "         'Dir',\n",
    "         'JerseyNumber',\n",
    "         'PlayerHeight',\n",
    "         'PlayerWeight',\n",
    "         'PlayerBMI',\n",
    "         'PlayerAge']\n",
    "\n",
    "    all_cols_player = np.array([[f'pl{num}_'+x for x in cols_player] for num in range(1, 23)]).flatten()\n",
    "\n",
    "    X = np.array(df[cols_player]).reshape(-1, len(cols_player)*22)\n",
    "\n",
    "    play_id_index = source_play_id[::22]\n",
    "    X_df = pd.DataFrame(X, columns=all_cols_player, index=play_id_index)\n",
    "\n",
    "    assert df[cols_player].shape[0] == X_df.shape[0] * 22\n",
    "    assert df[cols_player].shape[1] == X_df.shape[1] / 22\n",
    "\n",
    "    # Play features\n",
    "    cols_play = list(df.drop(cols_player+(['Yards'] if 'Yards' in df.columns else []), axis=1).columns)\n",
    "    X_play_col = np.zeros(shape=(X.shape[0], len(cols_play)))\n",
    "    for i, col in enumerate(cols_play):\n",
    "        X_play_col[:, i] = df[col][::22]\n",
    "\n",
    "    X_play_col_df = pd.DataFrame(X_play_col, columns=cols_play, index=play_id_index)\n",
    "    assert X_df.shape[0] == X_play_col_df.shape[0]\n",
    "    X_df = pd.concat([X_df, X_play_col_df], axis=1)\n",
    "\n",
    "    assert X_df.shape[0] == source_play_id.drop_duplicates().count()\n",
    "    return X_df\n",
    "\n",
    "def make_y(X, df):\n",
    "    y = np.zeros(shape=(X.shape[0], 199))\n",
    "    for i, yard in enumerate(df['Yards'][::22]):\n",
    "        y[i, yard+99:] = np.ones(shape=(1, 100-yard))\n",
    "    return y\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})\n",
    "df_train = pd.read_csv('data/train.csv', dtype={'WindSpeed': 'object'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_preprocessed = preprocess_features(df_train, fillna=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = make_x(df_train_preprocessed)\n",
    "y_train = make_y(X_train, df_train_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((23171, 313), (23171, 199))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_FEATURES = X_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterate_minibatches(X, y, batchsize):\n",
    "    X = torch.as_tensor(X)\n",
    "    y = torch.as_tensor(y)\n",
    "    indices = np.random.permutation(np.arange(len(X)))\n",
    "    for start in range(0, len(indices), batchsize):\n",
    "        ix = indices[start: start + batchsize]\n",
    "        yield X[ix], y[ix]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVICE = 'cpu'\n",
    "BATCH_SIZE = 32\n",
    "LEARNING_RATE = 0.002\n",
    "NUM_EPOCHS = 100\n",
    "EARLY_STOP_AFTER = 10\n",
    "WEIGHT_DECAY = 0.0002\n",
    "SCHEDULER_PATIENCE = 5\n",
    "L1_LOSS_WEIGHT = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.seq = nn.Sequential(\n",
    "            nn.Linear(N_FEATURES, N_FEATURES),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.1),\n",
    "            nn.BatchNorm1d(N_FEATURES),\n",
    "            nn.Linear(N_FEATURES, N_FEATURES),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(N_FEATURES),\n",
    "            nn.Linear(N_FEATURES, N_FEATURES),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.1),\n",
    "            nn.BatchNorm1d(N_FEATURES),\n",
    "        )\n",
    "        self.head_classifier = nn.Sequential(\n",
    "            nn.Linear(N_FEATURES, 199),\n",
    "            nn.Softmax()\n",
    "        )\n",
    "        self.head_regressor = nn.Sequential(\n",
    "            nn.Linear(N_FEATURES, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, x_batch):\n",
    "        latent_out = self.seq.forward(x_batch)\n",
    "        return self.head_classifier(latent_out), self.head_regressor(latent_out)\n",
    "\n",
    "    def loss(self, x_batch, y_batch):\n",
    "        y_batch = torch.as_tensor(y_batch).to(device=DEVICE)\n",
    "        y_pdf, y_yard = self(x_batch)\n",
    "        loss_crps = crps_loss(y_batch, y_pdf)\n",
    "        loss_mae = torch.mean(torch.abs(cdf_to_yard_torch(y_batch)-y_yard))\n",
    "        return loss_crps + loss_mae*L1_LOSS_WEIGHT\n",
    "\n",
    "    def predict_pdf(self, x_batch):\n",
    "        y_pdf, y_yard = self(x_batch)\n",
    "        return y_pdf\n",
    "    \n",
    "    def predict_cdf(self, x_batch):\n",
    "        y_pred_pdf = self.predict_pdf(x_batch)\n",
    "        y_pred_pdf = y_pred_pdf.cpu().data\n",
    "        y_pred_cdf = torch.cumsum(y_pred_pdf, dim=1).data.numpy()\n",
    "        y_pred_cdf = np.round(np.clip(y_pred_cdf, 0, 1), 10)\n",
    "        return y_pred_cdf\n",
    "\n",
    "model = NeuralNet()\n",
    "model = model.double().to(device=DEVICE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_patience, batch_size, n_epochs):\n",
    "    train_loss = []\n",
    "    train_loss_stds = []\n",
    "    val_accuracy = []\n",
    "    val_accuracy_stds = []\n",
    "\n",
    "    best_val_acc = None\n",
    "    best_model = None\n",
    "    patience = 0\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        try:\n",
    "            opt.zero_grad()\n",
    "\n",
    "            start_time = time.time()\n",
    "            model.train(True)\n",
    "            epoch_loss = []\n",
    "\n",
    "            for X_batch, y_batch in iter(iterate_minibatches(X_train, y_train, batch_size)):\n",
    "                loss = model.loss(X_batch, y_batch)\n",
    "                epoch_loss.append(float(loss.item()))\n",
    "                loss.backward()\n",
    "                opt.step()\n",
    "                opt.zero_grad()\n",
    "\n",
    "            train_loss.append(np.mean(epoch_loss))\n",
    "            train_loss_stds.append(np.std(epoch_loss))\n",
    "            model.train(False)\n",
    "\n",
    "            epoch_val_acc = []\n",
    "            for X_batch, y_batch in iter(iterate_minibatches(X_val, y_val, batch_size)): \n",
    "                y_val_pred_cdf = model.predict_cdf(X_batch)\n",
    "                val_loss = crps_loss_cdf(y_batch, y_val_pred_cdf)\n",
    "                epoch_val_acc.append(val_loss.item())\n",
    "            val_acc = np.mean(epoch_val_acc)\n",
    "            val_accuracy.append(val_acc)\n",
    "            val_accuracy_stds.append(np.std(epoch_val_acc))\n",
    "\n",
    "\n",
    "            scheduler.step(val_acc)\n",
    "\n",
    "            # Then we print the results for this epoch:\n",
    "            print(\"Epoch {} of {} took {:.3f}s\".format(\n",
    "                epoch + 1, n_epochs, time.time() - start_time))\n",
    "            print(\"  training loss: \\t{:.6f}\".format(train_loss[-1]))\n",
    "            print(\"  validation score: \\t\\t\\t{:.6f}\".format(val_accuracy[-1]))\n",
    "\n",
    "            if best_val_acc is None or val_acc < best_val_acc:\n",
    "                best_val_acc = val_acc\n",
    "                patience = 0\n",
    "                torch.save(model.state_dict(), 'best_model')\n",
    "            else:\n",
    "                patience += 1\n",
    "                print(f'Validation score has not improved for {patience} epochs.')\n",
    "            if patience >= early_stop_patience:\n",
    "                print('Early stopping.')\n",
    "                break\n",
    "        except KeyboardInterrupt:\n",
    "            break\n",
    "    model.load_state_dict(torch.load('best_model'))\n",
    "    model.train(False)\n",
    "\n",
    "    history = {\n",
    "        'train_loss': train_loss,\n",
    "        'train_loss_stds': train_loss_stds,\n",
    "        'val_accuracy': val_accuracy,\n",
    "        'val_accuracy_stds': val_accuracy_stds,\n",
    "    }\n",
    "    \n",
    "    return model, history\n",
    "\n",
    "def train_nn_pipeline(model, X, y, early_stop_after=EARLY_STOP_AFTER, batch_size=BATCH_SIZE, n_epochs=NUM_EPOCHS):\n",
    "    scaler = MinMaxScaler()\n",
    "    X = scaler.fit_transform(X)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1)\n",
    "    \n",
    "    opt = torch.optim.Adamax(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n",
    "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', patience=SCHEDULER_PATIENCE, verbose=True, threshold=1e-6, eps=1e-9)\n",
    "\n",
    "    model, train_history = train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_after, batch_size, n_epochs)\n",
    "    \n",
    "    return model, scaler, train_history\n",
    "\n",
    "def nn_pipeline_predict(model, scaler, X):\n",
    "    X = torch.as_tensor(scaler.transform(X)).to(device=DEVICE)\n",
    "    y_pred = model.predict_cdf(X)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 of 100 took 17.508s\n",
      "  training loss: \t0.018190\n",
      "  validation score: \t\t\t0.013877\n"
     ]
    }
   ],
   "source": [
    "model, scaler, history = train_nn_pipeline(model, X_train, y_train, n_epochs=NUM_EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEICAYAAADiGKj0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df3RU9Z3/8ddkhgmkhEziwiSLQ7QSPLWJSDfHxoLdsxMnAWKAQLLndC0rVsquUEzWLrira8BQQDGnIW63GpaFqlW2myrkkMETSYIGbLbWSjcFYTewRJOVDEeYAFHKJOP9/uG3U6cRhhh+fAjPxzmew733fT/zufd9wnk5n3uDzbIsSwAAADBW3JWeAAAAAM6PwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAHxGOBzWlClT9MEHHwz63Pfee08333zzJZgVgGud40pPAACGYsqUKZE/nzlzRk6nU3a7XZL0+OOPa9asWYMaz263a+/evRd1jgAwVAQ2AFe1z4Yrr9erH/zgB/rGN75xzvr+/n45HPzVB+Dqwt9aAIa1qqoqvffee4qLi9OuXbv02GOP6cYbb9TatWv1v//7vxo5cqSmT5+uhx9+WCNGjFB/f7+++tWvqqmpSddff73+/u//Xi6XSx0dHfr1r3+tSZMmqbKyUh6PJ+Znd3d3a8WKFdq7d69cLpcWLVqk4uJiSdJvfvMbPf7443rvvfc0cuRIzZ49Ww8//LDOnDmjf/qnf9KePXsUDod1ww03aMOGDUpJSbnUtwqAwXiGDcCw19jYqLvvvlu//vWvNXPmTNntdj366KP6z//8T23ZskW7d+/Wz372s3Oev337dpWWluqtt95SWlqaqqurL+hzy8rKdP3112v37t2qqqrSU089pbfeekuStGrVKt1///1655139Nprr2n69OmSpFdeeUVnzpzRG2+8oV/+8pdasWKF4uPjh34TAFzVCGwAhr2vfe1r8nq9iouL08iRI3Xrrbdq8uTJcjgc8ng8+su//MtIkPo8+fn5ysrK0ogRI1RYWKiDBw/G/MzOzk799re/1fe//33Fx8frq1/9qubOnau6ujpJ0ogRI9TR0aFgMKjRo0dr8uTJkiSHw6FgMKj33ntPdrtdWVlZ+tKXvnRxbgSAqxaBDcCwl5aWFrV9+PBhLVq0SFOnTtXXvvY1Pf300woGg+c8f+zYsZE/jxo1Sh9//HHMzzx27JiSk5OVkJAQ2Td+/HgFAgFJ0po1a3T48GFNnz5dxcXFeuONNyRJRUVF+sY3vqGysjLdeeedqqysVH9//6CuF8DwQ2ADMOzZbLao7RUrVigjI0Ovvfaa3nnnHT344IMX/TPHjRunYDAYFe4++OADud1uSdKXv/xlVVVVqbW1Vd/5zne0dOlSnT17Vk6nU0uXLtWrr76ql156STt37tT27dsv+vwAXF0IbACuOR999JESExOVkJCgw4cPn/f5tS/K4/EoMzNTP/zhDxUKhXTgwAG98sorkV8zsm3bNp04cUJxcXEaPXq0bDabbDabWltb9T//8z/65JNPNHr0aI0YMWJA4ARw7eEtUQDXnIcfflgrVqxQTU2NbrnlFs2YMUPvvPPORf+cqqoqrVixQlOnTpXL5dJDDz2kr3/965KklpYWPfHEEzp79qzGjx+vqqoqOZ1OHTt2TCtXrtSxY8eUkJCgmTNnqrCw8KLPDcDVxWZZlnWlJwEAAIBzY0kUAADAcAQ2AAAAwxHYAAAADEdgAwAAMNywfkv0k08+UTjMOxWDYbfbuGeGoSfmoSdmoi/moSeDM2KE/ZzHhnVgC4ct9fTE/o3k+AOXK4F7Zhh6Yh56Yib6Yh56Mjhjxyae8xhLogAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhLiiwtbS0KD8/Xz6fTxs2bBhwPBQKqaysTD6fTyUlJerq6pIkBYNBzZ8/X1OmTFFFRUXUOTt27FBhYaEKCgr01FNPxRxLkmpqauTz+ZSfn6/du3d/oQsGAAC42sQMbOFwWBUVFdq4caP8fr/q6+t16NChqJra2lqNGTNGO3fu1IIFC1RZWSlJio+PV2lpqZYvXx5VHwwGtW7dOj333HPy+/368MMP1draet6xDh06JL/fL7/fr40bN+rxxx9XOBy+KDcBAADAZDEDW1tbm9LT0+XxeOR0OlVQUKCmpqaomubmZhUVFUmS8vPz1draKsuylJCQoOzsbMXHx0fVd3Z2Kj09XSkpKZKkO+64Qw0NDecdq6mpSQUFBXI6nfJ4PEpPT1dbW9vQ7wAAAIDhHLEKAoGAUlNTI9tut3tAUAoEAkpLS/t0QIdDiYmJCgaDkUD2x9LT03XkyBF1dXUpNTVVTU1N6uvrO+9YgUBAkydPjppHIBA479ztdptcroRYl4jPsNvjuGeGoSfmoSdmoi/moScXT8zAdikkJSVp5cqV+ru/+zvFxcVpypQpev/99y/654TDlnp6Pr7o4w5nLlcC98ww9MQ89MRM9MU89GRwxo5NPOexmIHN7Xaru7s7sh0IBOR2uwfUHD16VKmpqerv79fp06eVnJx83nG9Xq+8Xq8k6Wc/+5ni4uLOO9aFzAMAAGA4ivkMW1ZWljo6OtTZ2alQKCS/3x8JWr/n9Xq1detWSVJDQ4NycnJks9nOO+7x48clSSdPntRLL72kkpKS847l9Xrl9/sVCoXU2dmpjo4O3XrrrYO/YgAAgKtMzG/YHA6HysvLtXDhQoXDYc2bN08ZGRmqrq5WZmamcnNzVVxcrGXLlsnn8ykpKUlVVVWR871er3p7e9XX16fGxkZt2rRJEydO1OrVq3Xw4EFJ0pIlS3TjjTdK0jnHysjI0IwZMzRz5kzZ7XaVl5fLbrdfinsCAABgFJtlWdaVnsSl0tcXZu18kHjewDz0xDz0xEz0xTz0ZHDO9wwb/9IBAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIZzXEhRS0uLVq9erU8++UQlJSVatGhR1PFQKKTly5dr//79crlcqqqq0vXXX69gMKgHH3xQ+/btU1FRkcrLyyPn1NfXq6amRpI0btw4PfXUU0pJSVFZWZmOHDkiSTp9+rQSExNVV1enrq4uzZw5UzfeeKMkafLkyaqoqLgoNwEAAMBkMQNbOBxWRUWFNm/eLLfbreLiYnm9Xk2cODFSU1tbqzFjxmjnzp3y+/2qrKzU+vXrFR8fr9LSUrW3t6u9vT1S39/fr9WrV8vv9yslJUXr1q3Tiy++qKVLl2r9+vWRuieeeEKjR4+ObE+YMEF1dXUX69oBAACuCjGXRNva2pSeni6PxyOn06mCggI1NTVF1TQ3N6uoqEiSlJ+fr9bWVlmWpYSEBGVnZys+Pj6q3rIsWZalM2fOyLIs9fb2aty4cQNqXn31Vd19991DvUYAAICrWsxv2AKBgFJTUyPbbrdbbW1tA2rS0tI+HdDhUGJiooLBoFJSUj53zBEjRmjlypUqLCxUQkKC0tPTtWLFiqiat99+W9ddd51uuOGGyL6uri7NmTNHo0ePVllZmbKzs887d7vdJpcrIdYl4jPs9jjumWHoiXnoiZnoi3noycVzQc+wXWx9fX3asmWLtm3bJo/Ho1WrVqmmpkaLFy+O1NTX10d9uzZu3Djt2rVLycnJ2rdvn5YsWSK/3x+1ZPrHwmFLPT0fX9JrGW5crgTumWHoiXnoiZnoi3noyeCMHZt4zmMxl0Tdbre6u7sj24FAQG63e0DN0aNHJX36fNrp06eVnJx8zjEPHDgg6dNn0mw2m2bMmKG9e/dGjvf392vnzp2aOXNmZJ/T6YyMmZmZqQkTJkReTgAAABjOYga2rKwsdXR0qLOzU6FQSH6/X16vN6rG6/Vq69atkqSGhgbl5OTIZrOdc0y3263Dhw/rxIkTkqQ333xTN910U+T4L37xC335y1+OWoo9ceKEwuGwJKmzs1MdHR3yeDyDuFQAAICrU8wlUYfDofLyci1cuFDhcFjz5s1TRkaGqqurlZmZqdzcXBUXF2vZsmXy+XxKSkpSVVVV5Hyv16ve3l719fWpsbFRmzZt0sSJE7VkyRLdc889cjgcGj9+vNauXRs5Z8eOHSooKIiax69+9Ss9/fTTcjgciouL0+OPPy6Xy3URbwUAAICZbJZlWVd6EpdKX1+YtfNB4nkD89AT89ATM9EX89CTwRnSM2wAAAC4sghsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIa7oMDW0tKi/Px8+Xw+bdiwYcDxUCiksrIy+Xw+lZSUqKurS5IUDAY1f/58TZkyRRUVFVHn1NfXq7CwUIWFhbr//vt14sQJSdI///M/684779Ts2bM1e/ZsvfHGG5Fzampq5PP5lJ+fr927d3/hiwYAALiaxAxs4XBYFRUV2rhxo/x+v+rr63Xo0KGomtraWo0ZM0Y7d+7UggULVFlZKUmKj49XaWmpli9fHlXf39+v1atX67nnntP27dt1880368UXX4wcX7Bggerq6lRXV6c///M/lyQdOnRIfr9ffr9fGzdu1OOPP65wODzkGwAAAGC6mIGtra1N6enp8ng8cjqdKigoUFNTU1RNc3OzioqKJEn5+flqbW2VZVlKSEhQdna24uPjo+oty5JlWTpz5owsy1Jvb6/GjRt33nk0NTWpoKBATqdTHo9H6enpamtrG+z1AgAAXHUcsQoCgYBSU1Mj2263e0BQCgQCSktL+3RAh0OJiYkKBoNKSUn53DFHjBihlStXqrCwUAkJCUpPT9eKFSsix1988UVt27ZNmZmZ+od/+AclJSUpEAho8uTJUfMIBALnnbvdbpPLlRDrEvEZdnsc98ww9MQ89MRM9MU89OTiiRnYLoW+vj5t2bJF27Ztk8fj0apVq1RTU6PFixfrW9/6lhYvXiybzabq6mo98cQTWrt27Rf6nHDYUk/Pxxd59sOby5XAPTMMPTEPPTETfTEPPRmcsWMTz3ks5pKo2+1Wd3d3ZDsQCMjtdg+oOXr0qKRPn087ffq0kpOTzznmgQMHJEkTJkyQzWbTjBkztHfvXknSn/zJn8hutysuLk4lJSX67W9/e8HzAAAAGI5iBrasrCx1dHSos7NToVBIfr9fXq83qsbr9Wrr1q2SpIaGBuXk5Mhms51zTLfbrcOHD0feDH3zzTd10003SZKOHTsWqWtsbFRGRkbkM/x+v0KhkDo7O9XR0aFbb711kJcLAABw9Ym5JOpwOFReXq6FCxcqHA5r3rx5ysjIUHV1tTIzM5Wbm6vi4mItW7ZMPp9PSUlJqqqqipzv9XrV29urvr4+NTY2atOmTZo4caKWLFmie+65Rw6HQ+PHj48sez711FM6ePCgJGn8+PGRXweSkZGhGTNmaObMmbLb7SovL5fdbr8U9wQAAMAoNsuyrCs9iUulry/M2vkg8byBeeiJeeiJmeiLeejJ4AzpGTYAAABcWQQ2AAAAwxHYAAAADEdgAwAAMByBDQAAwHAENgAAAMMR2AAAAAxHYAMAADAcgQ0AAMBwBDYAAADDEdgAAAAMR2ADAAAwHIENAADAcAQ2AAAAwxHYAAAADEdgAwAAMByBDQAAwHAENgAAAMNdUGBraWlRfn6+fD6fNmzYMOB4KBRSWVmZfD6fSkpK1NXVJUkKBoOaP3++pkyZooqKiqhz6uvrVVhYqMLCQt1///06ceKEJOnJJ5/U9OnTVVhYqCVLlujUqVOSpK6uLt16662aPXu2Zs+erfLy8iFdOAAAwNUiZmALh8OqqKjQxo0b5ff7VV9fr0OHDkXV1NbWasyYMdq5c6cWLFigyspKSVJ8fLxKS0u1fPnyqPr+/n6tXr1azz33nLZv366bb75ZL774oiRp6tSpqq+v1/bt23XDDTeopqYmct6ECRNUV1enurq6AQEQAABguIoZ2Nra2pSeni6PxyOn06mCggI1NTVF1TQ3N6uoqEiSlJ+fr9bWVlmWpYSEBGVnZys+Pj6q3rIsWZalM2fOyLIs9fb2aty4cZKkadOmyeFwSJJuu+02dXd3X5QLBQAAuFrFDGyBQECpqamRbbfbrUAgMKAmLS1NkuRwOJSYmKhgMHjOMUeMGKGVK1eqsLBQd955pw4fPqzi4uIBdS+//LK++c1vRra7uro0Z84cffvb39bbb78d++oAAACGAceV+NC+vj5t2bJF27Ztk8fj0apVq1RTU6PFixdHap555hnZ7XbNmjVLkjRu3Djt2rVLycnJ2rdvn5YsWSK/36/Ro0ef83PsdptcroRLfj3Did0exz0zDD0xDz0xE30xDz25eGIGNrfbHbUsGQgE5Ha7B9QcPXpUqamp6u/v1+nTp5WcnHzOMQ8cOCDp02fSJGnGjBlRLzO88sorev311/WTn/xENptNkuR0OuV0OiVJmZmZmjBhgo4cOaKsrKxzfk44bKmn5+NYl4jPcLkSuGeGoSfmoSdmoi/moSeDM3Zs4jmPxVwSzcrKUkdHhzo7OxUKheT3++X1eqNqvF6vtm7dKklqaGhQTk5OJGh9HrfbrcOHD0feDH3zzTd10003Sfr0jdSNGzfqmWee0ahRoyLnnDhxQuFwWJLU2dmpjo4OeTyeWNMHAAC46tksy7JiFb3xxhtas2aNwuGw5s2bpwceeEDV1dXKzMxUbm6uzp49q2XLlunAgQNKSkpSVVVVJEx5vV719vaqr69PiYmJ2rRpkyZOnKgtW7bo+eefl8Ph0Pjx47V27VolJyfL5/MpFArJ5XJJkiZPnqyKigo1NDTo6aeflsPhUFxcnJYuXTogOP6xvr4wyX6Q+L8h89AT89ATM9EX89CTwTnfN2wXFNiuVgS2weOHyzz0xDz0xEz0xTz0ZHCGtCQKAACAK4vABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhLiiwtbS0KD8/Xz6fTxs2bBhwPBQKqaysTD6fTyUlJerq6pIkBYNBzZ8/X1OmTFFFRUXUOfX19SosLFRhYaHuv/9+nThxQpLU09Oj++67T3l5ebrvvvt08uRJSZJlWfrBD34gn8+nwsJC7d+/f0gXDgAAcLWIGdjC4bAqKiq0ceNG+f1+1dfX69ChQ1E1tbW1GjNmjHbu3KkFCxaosrJSkhQfH6/S0lItX748qr6/v1+rV6/Wc889p+3bt+vmm2/Wiy++KEnasGGD7rjjDr322mu64447IgGxpaVFHR0deu2117Rq1SqtXLnyYlw/AACA8WIGtra2NqWnp8vj8cjpdKqgoEBNTU1RNc3NzSoqKpIk5efnq7W1VZZlKSEhQdnZ2YqPj4+qtyxLlmXpzJkzsixLvb29GjdunCSpqalJc+bMkSTNmTNHjY2NUfttNptuu+02nTp1SseOHRv6HQAAADCcI1ZBIBBQampqZNvtdqutrW1ATVpa2qcDOhxKTExUMBhUSkrK5445YsQIrVy5UoWFhUpISFB6erpWrFghSTp+/HgkvI0dO1bHjx//3HmkpqYqEAhEaj+P3W6Ty5UQ6xLxGXZ7HPfMMPTEPPTETPTFPPTk4okZ2C6Fvr4+bdmyRdu2bZPH49GqVatUU1OjxYsXR9XZbDbZbLYv/DnhsKWeno+HOt1risuVwD0zDD0xDz0xE30xDz0ZnLFjE895LOaSqNvtVnd3d2Q7EAjI7XYPqDl69KikT59PO336tJKTk8855oEDByRJEyZMkM1m04wZM7R3715J0nXXXRdZ6jx27FjkW7o/nkd3d/eAeQAAAAxHMQNbVlaWOjo61NnZqVAoJL/fL6/XG1Xj9Xq1detWSVJDQ4NycnLO+82Y2+3W4cOHI2+Gvvnmm7rpppsiY23btk2StG3bNuXm5kbttyxLv/nNb5SYmHje5VAAAIDhwmZZlhWr6I033tCaNWsUDoc1b948PfDAA6qurlZmZqZyc3N19uxZLVu2TAcOHFBSUpKqqqrk8XgkfRq0ent71dfXp8TERG3atEkTJ07Uli1b9Pzzz8vhcGj8+PFau3atkpOTFQwGVVZWpqNHj+pP//RPtX79erlcLlmWpYqKCu3evVujRo3SmjVrlJWVdd559/WF+Sp2kPj62jz0xDz0xEz0xTz0ZHDOtyR6QYHtakVgGzx+uMxDT8xDT8xEX8xDTwZnSM+wAQAA4MoisAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYznEhRS0tLVq9erU++eQTlZSUaNGiRVHHQ6GQli9frv3798vlcqmqqkrXX3+9gsGgHnzwQe3bt09FRUUqLy+XJPX29uqee+6JnN/d3a1Zs2bp0Ucf1Zo1a/TLX/5SkvS73/1Ox48f19tvvy1J+spXvqJJkyZJktLS0vTss88O/Q4AAAAYLmZgC4fDqqio0ObNm+V2u1VcXCyv16uJEydGamprazVmzBjt3LlTfr9flZWVWr9+veLj41VaWqr29na1t7dH6kePHq26urrI9ty5c5WXlydJeuSRRyL7X3jhBb377ruR7ZEjR0adBwAAcC2IuSTa1tam9PR0eTweOZ1OFRQUqKmpKaqmublZRUVFkqT8/Hy1trbKsiwlJCQoOztb8fHx5xz/yJEjOn78uLKzswcc8/v9uvvuuwd7TQAAAMNKzG/YAoGAUlNTI9tut1ttbW0DatLS0j4d0OFQYmKigsGgUlJSYk7A7/dr5syZstlsUfv/7//+T11dXcrJyYnsO3v2rObOnSuHw6FFixbprrvuOu/YdrtNLldCzDngD+z2OO6ZYeiJeeiJmeiLeejJxXNBz7BdSjt27NC6desG7Pf7/crPz5fdbo/s27Vrl9xutzo7O3Xvvfdq0qRJmjBhwjnHDoct9fR8fEnmPVy5XAncM8PQE/PQEzPRF/PQk8EZOzbxnMdiLom63W51d3dHtgOBgNxu94Cao0ePSpL6+/t1+vRpJScnx5zYwYMHFQ6HlZmZOeDYjh07VFBQMOBzJMnj8ej222+Per4NAABguIoZ2LKystTR0aHOzk6FQiH5/X55vd6oGq/Xq61bt0qSGhoalJOTM2CJ8/PU19cPCGWSdPjwYZ06dUpTpkyJ7Dt58qRCoZAk6cSJE3rnnXeiXnwAAAAYrmIuiTocDpWXl2vhwoUKh8OaN2+eMjIyVF1drczMTOXm5qq4uFjLli2Tz+dTUlKSqqqqIud7vV719vaqr69PjY2N2rRpUyRovfrqq9qwYcOAz9yxY8eA59oOHz6sFStWyGazybIsffe73yWwAQCAa4LNsizrSk/iUunrC7N2Pkg8b2AeemIeemIm+mIeejI4Q3qGDQAAAFcWgQ0AAMBwBDYAAADDEdgAAAAMR2ADAAAwHIENAADAcAQ2AAAAwxHYAAAADEdgAwAAMByBDQAAwHAENgAAAMMR2AAAAAxHYAMAADAcgQ0AAMBwBDYAAADDEdgAAAAMR2ADAAAwHIENAADAcBcU2FpaWpSfny+fz6cNGzYMOB4KhVRWViafz6eSkhJ1dXVJkoLBoObPn68pU6aooqIiUt/b26vZs2dH/vv617+u1atXS5JeeeUV5eTkRI7V1tZGztu6davy8vKUl5enrVu3DunCAQAArhaOWAXhcFgVFRXavHmz3G63iouL5fV6NXHixEhNbW2txowZo507d8rv96uyslLr169XfHy8SktL1d7ervb29kj96NGjVVdXF9meO3eu8vLyItszZ85UeXl51Dx6enr0ox/9SC+//LJsNpvmzp0rr9erpKSkId0AAAAA08X8hq2trU3p6enyeDxyOp0qKChQU1NTVE1zc7OKiookSfn5+WptbZVlWUpISFB2drbi4+PPOf6RI0d0/PhxZWdnn3cee/bs0dSpU+VyuZSUlKSpU6dq9+7dF3KNAAAAV7WY37AFAgGlpqZGtt1ut9ra2gbUpKWlfTqgw6HExEQFg0GlpKTEnIDf79fMmTNls9ki+1577TX96le/0o033qh//Md/VFpa2ufOIxAInHdsu90mlysh5hzwB3Z7HPfMMPTEPPTETPTFPPTk4okZ2C61HTt2aN26dZHtv/iLv9Ddd98tp9Opf//3f9fDDz+s559//guNHQ5b6un5+GJN9ZrgciVwzwxDT8xDT8xEX8xDTwZn7NjEcx6LuSTqdrvV3d0d2Q4EAnK73QNqjh49Kknq7+/X6dOnlZycHHNiBw8eVDgcVmZmZmRfcnKynE6nJKmkpET79++/4HkAAAAMRzEDW1ZWljo6OtTZ2alQKCS/3y+v1xtV4/V6I29tNjQ0KCcnJ2qJ81zq6+tVUFAQte/YsWORPzc3N+umm26SJE2bNk179uzRyZMndfLkSe3Zs0fTpk2LfYUAAABXuZhLog6HQ+Xl5Vq4cKHC4bDmzZunjIwMVVdXKzMzU7m5uSouLtayZcvk8/mUlJSkqqqqyPler1e9vb3q6+tTY2OjNm3aFHnD9NVXXx3wa0JeeOEFNTc3y263KykpSWvXrpUkuVwuLV68WMXFxZKkJUuWyOVyXbQbAQAAYCqbZVnWlZ7EpdLXF2btfJB43sA89MQ89MRM9MU89GRwhvQMGwAAAK4sAhsAAIDhCGwAAACGI0nOYoYAAAnFSURBVLABAAAYjsAGAABgOAIbAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOAIbAACA4RwXUtTS0qLVq1frk08+UUlJiRYtWhR1PBQKafny5dq/f79cLpeqqqp0/fXXKxgM6sEHH9S+fftUVFSk8vJySVJvb6/uueeeyPnd3d2aNWuWHn30UW3evFm1tbWy2+1KSUnRmjVrNH78eEnSV77yFU2aNEmSlJaWpmefffai3AQAAACTxQxs4XBYFRUV2rx5s9xut4qLi+X1ejVx4sRITW1trcaMGaOdO3fK7/ersrJS69evV3x8vEpLS9Xe3q729vZI/ejRo1VXVxfZnjt3rvLy8iR9GspefvlljRo1Si+99JKeeuoprV+/XpI0cuTIqPMAAACuBTGXRNva2pSeni6PxyOn06mCggI1NTVF1TQ3N6uoqEiSlJ+fr9bWVlmWpYSEBGVnZys+Pv6c4x85ckTHjx9Xdna2JCknJ0ejRo2SJN12223q7u7+whcHAAAwHMT8hi0QCCg1NTWy7Xa71dbWNqAmLS3t0wEdDiUmJioYDColJSXmBPx+v2bOnCmbzTbg2M9//nN985vfjGyfPXtWc+fOlcPh0KJFi3TXXXedd2y73SaXKyHmHPAHdnsc98ww9MQ89MRM9MU89OTiuaBn2C6lHTt2aN26dQP219XVad++ffrpT38a2bdr1y653W51dnbq3nvv1aRJkzRhwoRzjh0OW+rp+fiSzHu4crkSuGeGoSfmoSdmoi/moSeDM3Zs4jmPxVwSdbvdUcuSgUBAbrd7QM3Ro0clSf39/Tp9+rSSk5NjTuzgwYMKh8PKzMyM2v+LX/xCzz77rJ555hk5nc6oz5Ekj8ej22+/Xe+++27MzwAAALjaxQxsWVlZ6ujoUGdnp0KhkPx+v7xeb1SN1+vV1q1bJUkNDQ3Kycn53CXOP1ZfX6+CgoKofe+++67Ky8v1zDPP6LrrrovsP3nypEKhkCTpxIkTeuedd6JefAAAABiuYi6JOhwOlZeXa+HChQqHw5o3b54yMjJUXV2tzMxM5ebmqri4WMuWLZPP51NSUpKqqqoi53u9XvX29qqvr0+NjY3atGlTJGi9+uqr2rBhQ9TnrVu3Th9//LFKS0sl/eHXdxw+fFgrVqyQzWaTZVn67ne/S2ADAADXBJtlWdaVnsSl0tcXZu18kHjewDz0xDz0xEz0xTz0ZHCG9AwbAAAArqxh/Q0bAADAcMA3bAAAAIYjsAEAABiOwAYAAGA4AhsAAIDhCGwAAACGI7ABAAAYjsAGAABgOALbNaanp0f33Xef8vLydN999+nkyZOfW7d161bl5eUpLy8v8u/Eftbf/u3f6u67777U071mDKUvZ86c0aJFizR9+nQVFBSosrLyck592GlpaVF+fr58Pt+AfzpPkkKhkMrKyuTz+VRSUqKurq7IsZqaGvl8PuXn52v37t2Xc9rD2hftyZtvvqm5c+eqsLBQc+fOVWtr6+We+rA2lJ8VSfrggw80ZcoU/du//dvlmvLVzcI15cknn7Rqamosy7Ksmpoaa926dQNqgsGg5fV6rWAwaPX09Fher9fq6emJHG9oaLAeeughq6Cg4LLNe7gbSl8+/vhjq7W11bIsyzp79qz1rW99y3r99dcv6/yHi/7+fis3N9d6//33rbNnz1qFhYVWe3t7VM1Pf/pT67HHHrMsy7Lq6+ut0tJSy7Isq7293SosLLTOnj1rvf/++1Zubq7V399/2a9huBlKT/bv3291d3dblmVZ//3f/21Nmzbt8k5+GBtKX35v6dKl1tKlS62NGzdetnlfzfiG7RrT1NSkOXPmSJLmzJmjxsbGATV79uzR1KlT5XK5lJSUpKlTp0a+Lfjoo4+0efNmPfDAA5d13sPdUPoyatQo5eTkSJKcTqduueUWBQKByzr/4aKtrU3p6enyeDxyOp0qKChQU1NTVE1zc7OKiookSfn5+WptbZVlWWpqalJBQYGcTqc8Ho/S09PV1tZ2JS5jWBlKT2655Ra53W5JUkZGhs6ePatQKHTZr2E4GkpfJKmxsVHjx49XRkbGZZ/71YrAdo05fvy4xo0bJ0kaO3asjh8/PqAmEAgoNTU1su12uyMBoLq6Wt/5znc0cuTIyzPha8RQ+/J7p06d0q5du3THHXdc2gkPUxdyjwOBgNLS0iRJDodDiYmJCgaDF3QuBm8oPfmshoYG3XLLLXI6nZd+0teAofTlo48+0r/+67/qe9/73mWd89XOcaUngItvwYIF+vDDDwfsLysri9q22Wyy2WwXPO6BAwf0/vvv65FHHhnwLAJiu1R9+b3+/n499NBDmj9/vjwezxeeJzDctLe3q7KyUps2bbrSU4GkH/3oR7r33nv1pS996UpP5apCYBuGfvKTn5zz2HXXXadjx45p3LhxOnbsmFJSUgbUuN1uvfXWW5HtQCCg22+/XXv37tW+ffvk9XrV39+vEydOaP78+XrhhRcuxWUMO5eqL7/32GOP6YYbbtCCBQsu5rSvKW63W93d3ZHtQCAQWVL7bM3Ro0eVmpqq/v5+nT59WsnJyRd0LgZvKD2RpO7ubn3ve9/Tk08+qQkTJlzWuQ9nQ+nLf/3Xf6mhoUGVlZU6deqU4uLiFB8fr29/+9uX+zKuKiyJXmO8Xq+2bdsmSdq2bZtyc3MH1EybNk179uzRyZMndfLkSe3Zs0fTpk3TX/3VX2nPnj1qbm7WSy+9pBtuuIGwdpEMpS+SVFVVpd7eXj3yyCOXdd7DTVZWljo6OtTZ2alQKCS/3y+v1xtV4/V6I2/oNjQ0KCcnRzabTV6vV36/X6FQSJ2dnero6NCtt956JS5jWBlKT06dOqVFixbp+9//vv7sz/7sSkx/2BpKX1566SU1NzerublZ9957r/7mb/6GsHYhrvBLD7jMTpw4Yf31X/+15fP5rHvvvdcKBoOWZVlWW1ub9cgjj0Tqamtrrbvuusu66667rJ///OcDxuns7OQt0YtoKH05evSoNWnSJGv69OnWrFmzrFmzZln/8R//cUWuYzh4/fXXrby8PCs3N9f68Y9/bFmWZa1fv95qbGy0LMuyfve731lLly617rrrLmvevHnW+++/Hzn3xz/+sZWbm2vl5eXxpu5F9EV78i//8i/W5MmTIz8Xs2bNsj788MMrdh3DzVB+Vn7v6aef5i3RC2SzrP//ygYAAACMxJIoAACA4QhsAAAAhiOwAQAAGI7ABgAAYDgCGwAAgOEIbAAAAIYjsAEAABju/wGYKsVN05p9IwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAEICAYAAAD1Ojg9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df3RU9Z3/8deQMYOBkAQ2mcTdNBwW1h8HtKk5EhsW2xmGgTP8igGP5ZgVCItwrCWnGvY01mwbu1o0PTHb3SoYAuewRz1GCYuJGslEzOIJuq6cRnvYHthjSnJMJitMIITKZKb3+wfrfJtGmIRA+CR9Pv7yzucz7/l87vugL++9M9gsy7IEAACA627S9V4AAAAALiKYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAZg3Ojs7NTNN9+scDgsSdq4caPq6uqGNXekXnjhBT3++ONXvNZL2bdvn773ve9d9boAJgaCGYAxU1RUpKqqqiGvNzU1KS8vb8Qhqrq6Wvn5+aNe1wcffKCFCxcOem3z5s36p3/6p1HXBoCRIJgBGDP5+fk6cOCA/vR3rQ8cOKDly5fLbrdfp5UBgBkIZgDGzKJFi9Tb26uPPvoo+tqZM2f07rvvatWqVZKkQ4cOadWqVfrWt76le+65R7/85S8vWa+wsFC1tbWSpEgkou3bt2v+/Plyu9167733Bs19/fXXtXTpUmVnZ8vtduuVV16RJJ0/f15///d/r56eHmVnZys7O1uBQEC//OUv9dhjj0Xf7/f75fP5lJOTo8LCQv3P//xPdMzlcmnXrl1avny57rzzThUXF+vChQvDOicff/yxCgoKdOedd6qgoEAff/xxdGzfvn1yu93Kzs6Wy+XSgQMHJEm/+93v9MADD+jOO+/U/PnzVVxcPKzPAmA+ghmAMTN58mQtXbpU+/fvj7721ltvadasWbrlllskSTfeeKO2b9+ujz76SDt27NDLL7+spqammLVfffVVvfvuu9q/f79ef/11vf3224PGZ8yYoR07dujjjz/W008/raefflq/+c1vlJCQoBdffFFpaWk6evSojh49KqfTOei9n332mR599FGVlpaqtbVVCxcu1ObNmxUKhQbto7q6Wn6/X7/97W+1b9++mGvu7e3VQw89pMLCQn3wwQdav369HnroIQWDQZ0/f14/+9nP9OKLL+ro0aN65ZVXdOutt0qSqqqqlJeXp//8z/9US0uLHnjggZifBWB8IJgBGFOrVq1SY2Nj9IrS/v37Bz0nNn/+fN18882aNGmSbrnlFvl8Pn344Ycx67711lt68MEHlZGRoeTkZD300EODxr/zne/oG9/4hmw2m+666y7l5eUNunJ3OW+++abuuece5eXl6YYbblBRUZG+/PJLHT16NDqnsLBQTqdTycnJ+u53v6tjx47FrHvo0CFlZWVp1apVstvtWrZsmWbNmqV3331XkjRp0iQdP35cX375pdLS0jRnzhxJkt1u1+eff66enh45HA7l5OQMax8AzEcwAzCmcnJylJKSoqamJp08eVKffPKJli1bFh3/9a9/rcLCQuXm5urOO+/UK6+8omAwGLNuT0+PMjIyosc33XTToPH33ntP9913n+666y7l5OSopaVlWHW/qv3H9SZNmqSMjAwFAoHoa6mpqdF/vvHGG3X+/PkR1/1q3YFAQAkJCaqsrNQrr7yiBQsWaNOmTdHbpyUlJbIsS6tXr5bP59Nrr702rH0AMB/BDMCYW7lypfbv368DBw5owYIF+ou/+Ivo2KOPPhp9Ruy//uu/dP/99w/5ssDXSU1NVVdXV/T4j/85FArpBz/4gTZs2KD3339fH330kRYuXBita7PZLls7LS1Nn3/+efTYsix1dXUNueU5Un9a96t1f1X3b//2b7V7924dPnxYs2bN0hNPPBHd689+9jMdPnxYP/3pT/XTn/5Uv/vd70a1FgBmIJgBGHOrVq1Sa2urXn311ehD/1/p7+9XUlKSHA6H2traVF9fP6yaS5cu1d69e9Xd3a0zZ85o586d0bFQKKRQKKTp06fLbrfrvffe0/vvvx8dnzFjhnp7e9XX13fJ2u+9955aW1s1MDCgmpoaxcfHKzs7+wp2///dc889am9v1xtvvKFwOKw333xTJ06c0He+8x198cUXampq0vnz5xUfH6+EhARNmnTxX9lvvfWWuru7JUlJSUmy2WzRMQDjG99NBzDm/uqv/krZ2dn67//+b7nd7kFj//iP/6jt27ervLxcd911l5YuXaqzZ8/GrHnfffepvb1dK1eu1JQpU1RUVKQjR45IkqZOnaof//jHKi4uVigU0ne/+125XK7oe//6r/9aPp9PixYtUiQSUUNDw6Das2bN0rPPPqsnn3xSgUBAt956q1544QXFx8eP6jykpKTohRde0FNPPaWf/OQnysrK0gsvvKDp06erp6dHe/bs0T/8wz/IZrPp1ltv1U9+8hNJ0ieffKKnnnpK586d04wZM/T4448rMzNzVGsBYAabNZx7BAAAALjmuPYNAABgCIIZAACAIQhmAAAAhiCYAQAAGGJCfCvzD3/4gyIRvsMwXHFxNs6XYeiJmeiLeeiJmejLyNxwQ9wlxyZEMItELPX2xv6VbVyUnJzA+TIMPTETfTEPPTETfRmZ1NTES45xKxMAAMAQwwpmLS0t8nq98ng8g35N+yuhUEjFxcXyeDxas2aNOjs7JUnBYFCFhYXKzs5WeXn519bevHnzoL8nT5L27t2rJUuWyOfz6ZlnnhnpngAAAMalmLcyI5GIysvLtXv3bjmdTq1evVoul0uzZ8+OzqmtrdW0adN08OBBNTQ0qKKiQs8995wcDoe2bt2q48eP6/jx40Nqv/POO5oyZcqg144cOSK/368DBw4oPj5ep06dugrbBAAAMF/MK2ZtbW3KyspSZmam4uPj5fP55Pf7B81pbm5Wfn6+JMnr9aq1tVWWZSkhIUE5OTlyOBxD6vb392v37t3asmXLoNdffvllbdq0KfpXncyYMeOKNwcAADCexLxiFggElJ6eHj12Op1qa2sbMicjI+NiQbtdiYmJCgaDmj59+iXrVlVVacOGDZo8efKg19vb2/XRRx+psrJSDodD27Zt0+23337ZNcbF2ZScnBBrK/g/cXGTOF+GoSdmoi/moSdmoi9Xz3X5VuaxY8d08uRJlZaWRp9H+0okEtGZM2f06quv6pNPPlFxcbH8fr9sNtsl6/GtzJHh2zPmoSdmoi/moSdmoi8jM6pvZTqdTnV3d0ePA4GAnE7nkDldXV2SpHA4rL6+PqWkpFyy5tGjR/Xpp5/K5XJp7dq1am9vV2FhYbSWx+ORzWbT7bffrkmTJikYDMZaJgAAwLgXM5jNmzdP7e3t6ujoUCgUUkNDg1wu16A5LpdLdXV1kqTGxkbl5uZe9grX2rVrdfjwYTU3N+ull17SzJkztXfvXknSokWL9MEHH0iSPvvsMw0MDFw25AEAAEwUMW9l2u12lZWVaePGjYpEIiooKNCcOXNUVVWluXPnyu12a/Xq1SopKZHH41FSUpIqKyuj73e5XDp37pwGBgbU1NSkmpqaQd/o/FMFBQUqLS3VsmXLdMMNN+jnP//5ZUMeAADARGGzLGvc/x0KAwMR7m2PAM8CmIeemIm+mIeemIm+jAy//A8AADAOEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMMSwgllLS4u8Xq88Ho927tw5ZDwUCqm4uFgej0dr1qxRZ2enJCkYDKqwsFDZ2dkqLy//2tqbN2/WsmXLhrxeU1Ojm2++WadPnx7JfgAAAMatmMEsEomovLxc1dXVamhoUH19vU6cODFoTm1traZNm6aDBw9q3bp1qqiokCQ5HA5t3bpV27Zt+9ra77zzjqZMmTLk9a6uLr3//vu66aabrmRPAAAA41LMYNbW1qasrCxlZmYqPj5ePp9Pfr9/0Jzm5mbl5+dLkrxer1pbW2VZlhISEpSTkyOHwzGkbn9/v3bv3q0tW7YMGXv66adVUlIim812pfsCAAAYd2IGs0AgoPT09Oix0+lUIBAYMicjI0OSZLfblZiYqGAweNm6VVVV2rBhgyZPnjzo9aamJqWlpemWW24Z9iYAAAAmAvv1+NBjx47p5MmTKi0tjT6PJkm///3vtWPHDtXU1IyoXlycTcnJCVd7mRNWXNwkzpdh6ImZ6It56ImZ6MvVEzOYOZ1OdXd3R48DgYCcTueQOV1dXUpPT1c4HFZfX59SUlIuWfPo0aP69NNP5XK5FA6Hdfr0aRUWFurHP/6xOjs7tXLlSklSd3e37r33XtXW1io1NfWS9SIRS72952NuFhclJydwvgxDT8xEX8xDT8xEX0YmNTXxkmMxg9m8efPU3t6ujo4OOZ1ONTQ06Be/+MWgOS6XS3V1dcrOzlZjY6Nyc3Mv+3zY2rVrtXbtWklSZ2enNm/erL1790qSWltbB9V97bXXNH369FjLBAAAGPdiBjO73a6ysjJt3LhRkUhEBQUFmjNnjqqqqjR37ly53W6tXr1aJSUl8ng8SkpKUmVlZfT9LpdL586d08DAgJqamlRTU6PZs2df000BAACMRzbLsqzrvYjRGhiIcAl1BLjkbB56Yib6Yh56Yib6MjKXu5XJL/8DAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYYVjBraWmR1+uVx+PRzp07h4yHQiEVFxfL4/FozZo16uzslCQFg0EVFhYqOztb5eXlX1t78+bNWrZsWfR4+/btWrJkiZYvX66HH35YZ8+evZJ9AQAAjDsxg1kkElF5ebmqq6vV0NCg+vp6nThxYtCc2tpaTZs2TQcPHtS6detUUVEhSXI4HNq6dau2bdv2tbXfeecdTZkyZdBreXl5qq+v1xtvvKGZM2dqx44dV7o3AACAcSVmMGtra1NWVpYyMzMVHx8vn88nv98/aE5zc7Py8/MlSV6vV62trbIsSwkJCcrJyZHD4RhSt7+/X7t379aWLVsGvb5gwQLZ7XZJ0je/+U11d3df8eYAAADGE3usCYFAQOnp6dFjp9Optra2IXMyMjIuFrTblZiYqGAwqOnTp1+yblVVlTZs2KDJkydfcs7rr7+upUuXxtxEXJxNyckJMefhori4SZwvw9ATM9EX89ATM9GXqydmMLsWjh07ppMnT6q0tDT6PNqfev755xUXF6cVK1bErBeJWOrtPX+1lzlhJScncL4MQ0/MRF/MQ0/MRF9GJjU18ZJjMYOZ0+kcdDsxEAjI6XQOmdPV1aX09HSFw2H19fUpJSXlkjWPHj2qTz/9VC6XS+FwWKdPn1ZhYaH27t0rSdq3b58OHTqkPXv2yGazxdwgAADARBDzGbN58+apvb1dHR0dCoVCamhokMvlGjTH5XKprq5OktTY2Kjc3NzLBqq1a9fq8OHDam5u1ksvvaSZM2dGQ1lLS4uqq6v1/PPP68YbbxzN3gAAAMaVmFfM7Ha7ysrKtHHjRkUiERUUFGjOnDmqqqrS3Llz5Xa7tXr1apWUlMjj8SgpKUmVlZXR97tcLp07d04DAwNqampSTU2NZs+efcnPe/LJJxUKhbR+/XpJ0h133HHJn9oAAACYSGyWZVnXexGjNTAQ4d72CPAsgHnoiZnoi3noiZnoy8hc7hkzfvkfAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAwxrGDW0tIir9crj8ejnTt3DhkPhUIqLi6Wx+PRmjVr1NnZKUkKBoMqLCxUdna2ysvLv7b25s2btWzZsuhxb2+v1q9fr8WLF2v9+vU6c+bMlewLAABg3IkZzCKRiMrLy1VdXa2GhgbV19frxIkTg+bU1tZq2rRpOnjwoNatW6eKigpJksPh0NatW7Vt27avrf3OO+9oypQpg17buXOn7r77br3zzju6++67vzYIAgAATEQxg1lbW5uysrKUmZmp+Ph4+Xw++f3+QXOam5uVn58vSfJ6vWptbZVlWUpISFBOTo4cDseQuv39/dq9e7e2bNky6HW/369Vq1ZJklatWqWmpqYr3hwAAMB4EjOYBQIBpaenR4+dTqcCgcCQORkZGZIku92uxMREBYPBy9atqqrShg0bNHny5EGvnzp1SmlpaZKk1NRUnTp1ang7AQAAGOfs1+NDjx07ppMnT6q0tDT6PNrXsdlsstlsMevFxdmUnJxwNZc4ocXFTeJ8GYaemIm+mIeemIm+XD0xg5nT6VR3d3f0OBAIyOl0DpnT1dWl9PR0hcNh9fX1KSUl5ZI1jx49qk8//VQul0vhcFinT59WYWGh9u7dqxkzZqinp0dpaWnq6enR9OnTY24iErHU23s+5jxclJycwPkyDD0xE30xDz0xE30ZmdTUxEuOxbyVOW/ePLW3t6ujo0OhUEgNDQ1yuVyD5rhcLtXV1UmSGhsblZube9krXWvXrtXhw4fV3Nysl156STNnztTevXujtfbv3y9J2r9/v9xud+wdAgAATAAxr5jZ7XaVlZVp48aNikQiKigo0Jw5c1RVVaW5c+fK7XZr9erVKikpkcfjUVJSkiorK6Pvd7lcOnfunAYGBtTU1KSamhrNnj37kp+3adMmFRcX67XXXtNNN92k55577ursFAAAwHA2y7Ks672I0RoYiHAJdQS45GweemIm+mIeemIm+jIyo7qVCQAAgLFBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQwwrmLW0tMjr9crj8Wjnzp1DxkOhkIqLi+XxeLRmzRp1dnZKkoLBoAoLC5Wdna3y8vJB7ykqKtKKFSvk8/lUVlamSCQiSTp27Jjuu+8+rVy5Uvfee6/a2tpGu0cAAIBxIWYwi0QiKi8vV3V1tRoaGlRfX68TJ04MmlNbW6tp06bp4MGDWrdunSoqKiRJDodDW7du1bZt24bUraqq0oEDB1RfX69gMKi3335bkvTss8/q4Ycf1r//+79r69atevbZZ6/GPgEAAIwXM5i1tbUpKytLmZmZio+Pl8/nk9/vHzSnublZ+fn5kiSv16vW1lZZlqWEhATl5OTI4XAMqTt16lRJUjgc1sDAgGw2myTJZrOpv79fktTX16e0tLTR7RAAAGCcsMeaEAgElJ6eHj12Op1Dbi8GAgFlZGRcLGi3KzExUcFgUNOnT79s7aKiIrW1tWnhwoXyer2SpNLSUhUVFWn79u36wx/+oFdeeSXmJuLibEpOTog5DxfFxU3ifBmGnpiJvpiHnpiJvlw9MYPZtbRr1y5duHBBjz32mI4cOaK8vDy9/PLL+tGPfiSv16s333xTjz/+uPbs2XPZOpGIpd7e82Oz6AkgOTmB82UYemIm+mIeemIm+jIyqamJlxyLeSvT6XSqu7s7ehwIBOR0OofM6erqknTx1mRfX59SUlKGtTiHwyG32x29PVpXV6fFixdLkpYuXcrD/wAA4M9GzGA2b948tbe3q6OjQ6FQSA0NDXK5XIPmuFwu1dXVSZIaGxuVm5sbfWbs6/T396unp0fSxSB36NAhzZo1S5KUlpamDz/8UJJ05MgRzZw584o2BgAAMN7EvJVpt9tVVlamjRs3KhKJqKCgQHPmzFFVVZXmzp0rt9ut1atXq6SkRB6PR0lJSaqsrIy+3+Vy6dy5cxoYGFBTU5NqamqUnJysLVu2KBQKybIszZ8/X/fff78k6cknn9RTTz2lcDgsh8Mx5Gc2AAAAJiqbZVnW9V7EaA0MRLi3PQI8C2AeemIm+mIeemIm+jIyo3rGDAAAAGODYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIYYVzFpaWuT1euXxeLRz584h46FQSMXFxfJ4PFqzZo06OzslScFgUIWFhcrOzlZ5efmg9xQVFWnFihXy+XwqKytTJBKJju3du1dLliyRz+fTM888M5r9AQAAjBv2WBMikYjKy8u1e/duOZ1OrV69Wi6XS7Nnz47Oqa2t1bRp03Tw4EE1NDSooqJCzz33nBwOh7Zu3arjx4/r+PHjg+pWVVVp6tSpsixLP/jBD/T222/L5/PpyJEj8vv9OnDggOLj43Xq1Kmrv2sAAAADxbxi1tbWpqysLGVmZio+Pl4+n09+v3/QnObmZuXn50uSvF6vWltbZVmWEhISlJOTI4fDMaTu1KlTJUnhcFgDAwOy2WySpJdfflmbNm1SfHy8JGnGjBmj2yEAAMA4ETOYBQIBpaenR4+dTqcCgcCQORkZGZIku92uxMREBYPBmB9eVFSkb3/725oyZYq8Xq8kqb29XR999JHWrFmjBx54QG1tbSPaEAAAwHgV81bmtbRr1y5duHBBjz32mI4cOaK8vDxFIhGdOXNGr776qj755BMVFxfL7/dHr6h9nbg4m5KTE8Zw5eNbXNwkzpdh6ImZ6It56ImZ6MvVEzOYOZ1OdXd3R48DgYCcTueQOV1dXUpPT1c4HFZfX59SUlKGtQCHwyG32y2/36+8vDw5nU55PB7ZbDbdfvvtmjRpkoLBoKZPn37JGpGIpd7e88P6PEjJyQmcL8PQEzPRF/PQEzPRl5FJTU285FjMW5nz5s1Te3u7Ojo6FAqF1NDQIJfLNWiOy+VSXV2dJKmxsVG5ubmXvcLV39+vnp4eSRefMTt06JBmzZolSVq0aJE++OADSdJnn32mgYGBYYc8AACA8SzmFTO73a6ysjJt3LhRkUhEBQUFmjNnjqqqqjR37ly53W6tXr1aJSUl8ng8SkpKUmVlZfT9LpdL586d08DAgJqamlRTU6Pk5GRt2bJFoVBIlmVp/vz5uv/++yVJBQUFKi0t1bJly3TDDTfo5z//+WVDHgAAwERhsyzLut6LGK2BgQiXUEeAS87moSdmoi/moSdmoi8jM6pbmQAAABgbBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMMSwgllLS4u8Xq88Ho927tw5ZDwUCqm4uFgej0dr1qxRZ2enJCkYDKqwsFDZ2dkqLy8f9J6ioiKtWLFCPp9PZWVlikQig8Zramp088036/Tp01e6NwAAgHElZjCLRCIqLy9XdXW1GhoaVF9frxMnTgyaU1tbq2nTpungwYNat26dKioqJEkOh0Nbt27Vtm3bhtStqqrSgQMHVF9fr2AwqLfffjs61tXVpffff1833XTTaPcHAAAwbsQMZm1tbcrKylJmZqbi4+Pl8/nk9/sHzWlublZ+fr4kyev1qrW1VZZlKSEhQTk5OXI4HEPqTp06VZIUDoc1MDAgm80WHXv66adVUlIy6DUAAICJzh5rQiAQUHp6evTY6XSqra1tyJyMjIyLBe12JSYmKhgMavr06ZetXVRUpLa2Ni1cuFBer1eS1NTUpLS0NN1yyy3D3kRcnE3JyQnDnv/nLi5uEufLMPTETPTFPPTETPTl6okZzK6lXbt26cKFC3rsscd05MgRfetb39KOHTtUU1MzojqRiKXe3vPXaJUTT3JyAufLMPTETPTFPPTETPRlZFJTEy85FvNWptPpVHd3d/Q4EAjI6XQOmdPV1SXp4q3Jvr4+paSkDGtxDodDbrdbfr9fJ0+eVGdnp1auXCmXy6Xu7m7de++9+t///d9h1QIAABjPYgazefPmqb29XR0dHQqFQmpoaJDL5Ro0x+Vyqa6uTpLU2Nio3Nzcyz4f1t/fr56eHkkXg9yhQ4c0a9Ys3XzzzWptbVVzc7Oam5uVnp6uffv2KTU1dTR7BAAAGBdi3sq02+0qKyvTxo0bFYlEVFBQoDlz5qiqqkpz586V2+3W6tWrVVJSIo/Ho6SkJFVWVkbf73K5dO7cOQ0MDKipqUk1NTVKTk7Wli1bFAqFZFmW5s+fr/vvv/+abhQAAMB0NsuyrOu9iNEaGIhwb3sEeBbAPPTETPTFPPTETPRlZEb1jBkAAADGBsEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMMK5i1tLTI6/XK4/Fo586dQ8ZDoZCKi4vl8Xi0Zs0adXZ2SpKCwaAKCwuVnZ2t8vLyQe8pKirSihUr5PP5VFZWpkgkIknavn27lixZouXLl+vhhx/W2bNnR7tHAACAcSFmMItEIiovL1d1dbUaGhpUX1+vEydODJpTW1uradOm6eDBg1q3bp0qKiokSQ6HQ1u3btW2bduG1K2qqtKBAwdUX1+vYDCot99+W5KUl5en+vp6vfHGG5o5c6Z27NhxNfYJAABgvJjBrK2tTVlZWcrMzFR8fLx8Pp/8fv+gOc3NzcrPz5ckeb1etba2yrIsJSQkKCcnRw6HY2OO5+sAAAgrSURBVEjdqVOnSpLC4bAGBgZks9kkSQsWLJDdbpckffOb31R3d/fodggAADBOxAxmgUBA6enp0WOn06lAIDBkTkZGhiTJbrcrMTFRwWAw5ocXFRXp29/+tqZMmSKv1ztk/PXXX9fChQtj1gEAAJgI7Nfzw3ft2qULFy7oscce05EjR5SXlxcde/755xUXF6cVK1bErBMXZ1NycsK1XOqEEhc3ifNlGHpiJvpiHnpiJvpy9cQMZk6nc9DtxEAgIKfTOWROV1eX0tPTFQ6H1dfXp5SUlGEtwOFwyO12y+/3R4PZvn37dOjQIe3Zsyd6i/NyIhFLvb3nh/V5kJKTEzhfhqEnZqIv5qEnZqIvI5OamnjJsZi3MufNm6f29nZ1dHQoFAqpoaFBLpdr0ByXy6W6ujpJUmNjo3Jzcy8bqPr7+9XT0yPp4jNmhw4d0qxZsyRd/AZodXW1nn/+ed14442xdwcAADBB2CzLsmJNeu+99/TUU08pEomooKBAW7ZsUVVVlebOnSu3260LFy6opKREx44dU1JSkiorK5WZmSnpYmg7d+6cBgYGlJiYqJqaGiUnJ+uhhx5SKBSSZVmaP3++fvSjH8lut8vj8SgUCik5OVmSdMcddwz5qY0/NTAQIamPAP9nYx56Yib6Yh56Yib6MjKXu2I2rGBmOoLZyPAHyDz0xEz0xTz0xEz0ZWRGdSsTAAAAY4NgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAYAAGAIghkAAIAhJsTvmAEAAEwEXDEDAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzCao3t5erV+/XosXL9b69et15syZr51XV1enxYsXa/HixaqrqxsyvnnzZi1btuxaL/fPwmh68vvf/16bNm3SkiVL5PP5VFFRMZZLn5BaWlrk9Xrl8Xi0c+fOIeOhUEjFxcXyeDxas2aNOjs7o2M7duyQx+OR1+vVf/zHf4zlsie0K+3J+++/r3vvvVfLly/Xvffeq9bW1rFe+oQ1mj8nkvT5558rOztbu3btGqslj38WJqTt27dbO3bssCzLsnbs2GE988wzQ+YEg0HL5XJZwWDQ6u3ttVwul9Xb2xsdb2xstH74wx9aPp9vzNY9kY2mJ+fPn7daW1sty7KsCxcuWN/73vesQ4cOjen6J5JwOGy53W7r5MmT1oULF6zly5dbx48fHzTn3/7t36wnnnjCsizLqq+vt7Zu3WpZlmUdP37cWr58uXXhwgXr5MmTltvttsLh8JjvYaIZTU9+85vfWN3d3ZZlWdZvf/tba8GCBWO7+AlqND35yiOPPGI98sgjVnV19Zite7zjitkE5ff7tWrVKknSqlWr1NTUNGTO4cOHlZeXp+TkZCUlJSkvLy/6f//9/f3avXu3tmzZMqbrnshG05Mbb7xRubm5kqT4+HjddtttCgQCY7r+iaStrU1ZWVnKzMxUfHy8fD6f/H7/oDnNzc3Kz8+XJHm9XrW2tsqyLPn9fvl8PsXHxyszM1NZWVlqa2u7HtuYUEbTk9tuu01Op1OSNGfOHF24cEGhUGjM9zDRjKYnktTU1KS//Mu/1Jw5c8Z87eMZwWyCOnXqlNLS0iRJqampOnXq1JA5gUBA6enp0WOn0xn9j31VVZU2bNigyZMnj82C/wyMtidfOXv2rN59913dfffd13bBE9hwznMgEFBGRoYkyW63KzExUcFgcFjvxciNpid/rLGxUbfddpvi4+Ov/aInuNH0pL+/Xy+++KK+//3vj+maJwL79V4Arty6dev0xRdfDHm9uLh40LHNZpPNZht23WPHjunkyZMqLS0d8rwALu9a9eQr4XBYP/zhD1VYWKjMzMwrXicwER0/flwVFRWqqam53kv5s/cv//IvevDBBzVlypTrvZRxh2A2ju3Zs+eSYzNmzFBPT4/S0tLU09Oj6dOnD5njdDr14YcfRo8DgYDuuusuHT16VJ9++qlcLpfC4bBOnz6twsJC7d2791psY0K5Vj35yhNPPKGZM2dq3bp1V3PZf3acTqe6u7ujx4FAIHor7I/ndHV1KT09XeFwWH19fUpJSRnWezFyo+mJJHV3d+v73/++tm/frm984xtjuvaJajQ9+fWvf63GxkZVVFTo7NmzmjRpkhwOhx544IGx3sa4w63MCcrlcmn//v2SpP3798vtdg+Zs2DBAh0+fFhnzpzRmTNndPjwYS1YsEBr167V4cOH1dzcrJdeekkzZ84klF0Fo+mJJFVWVurcuXMqLS0d03VPRPPmzVN7e7s6OjoUCoXU0NAgl8s1aI7L5Yp+K7axsVG5ubmy2WxyuVxqaGhQKBRSR0eH2tvbdfvtt1+PbUwoo+nJ2bNntWnTJj366KO68847r8fyJ6TR9OSll15Sc3Ozmpub9eCDD+qhhx4ilA3Xdf7yAa6R06dPW3/3d39neTwe68EHH7SCwaBlWZbV1tZmlZaWRufV1tZaixYtshYtWmS99tprQ+p0dHTwrcyrZDQ96erqsv7mb/7GWrJkibVixQprxYoV1quvvnpd9jFRHDp0yFq8eLHldrutX/3qV5ZlWdZzzz1nNTU1WZZlWV9++aX1yCOPWIsWLbIKCgqskydPRt/7q1/9ynK73dbixYv5duxVdKU9+dd//VfrjjvuiP7ZWLFihfXFF19ct31MJKP5c/KVf/7nf+ZbmSNgs6z/+/oEAAAArituZQIAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIf4fOH3taU5RgBoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAEICAYAAAD1Ojg9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df3SUZX7//9dkhgGRQAKFCeoYxI7dakKKe9QgyB8Th5FM+BEIpfbsViIc3B7FRjTLHqvZJq5Stykxbrtu2NhVa7W7UAJH5mwjTGADnOxuT6WNta5r7A4ONbnphgQJYRky3J8/+DJn8404xCTkSvb5+Cfc931d931d9/sEXtw/Zhy2bdsCAADAqEsb7QEAAADgIoIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgDGjUQiofnz5+uTTz4ZdN9jx47pD/7gD0ZgVMOvpqZG3/jGN0Z7GABGgGu0BwDgd9f8+fOTfz579qzcbrecTqckqbKyUsuXLx/U/pxOp44ePTqsY7zaampqZFmW/vqv/3q0hwJgFBDMAIya3w5Rfr9f3/rWt3T33Xdftn1fX59cLv7aAjB+8TccAGPV1NTo2LFjSktL04EDB/T000/rpptu0tatW/U///M/mjRpku677z5t2bJFEyZMUF9fn2677TZFIhHdcMMNeuKJJ5SRkaFoNKp///d/1y233KLq6mp5vd6Ux+7o6NA3v/lNHT16VBkZGdq4caNKSkokSf/xH/+hyspKHTt2TJMmTdKKFSu0ZcsWnT17Vk899ZQOHz6sRCKhOXPmaPv27Zo+ffqA/X/ve9/T66+/rt7eXs2aNUuVlZXq7e3Vyy+/LNu21djYqJtuukm7du1SLBbTli1b9P777+v222/XDTfcMOznGoAZCGYAjLZ//369+OKLqq6uVjwe1y9/+Uv95V/+pW677Ta1t7drw4YNmjNnjr7yla98Zv+33npL9fX1+tKXvqTy8nLV1taquro65XHLysp022236cUXX1RbW5sefPBB3Xjjjbrzzjv1zDPPaP369SoqKlJPT48++ugjSdKuXbt09uxZ/eQnP9GECRP03//935o4ceKAfX/44Yf64Q9/qIaGBs2cOVOxWEyS5PV6tX79+gG3MsvKynTnnXfqlVde0dGjR/W1r31NwWDwi5xOAIbj4X8ARrv99tvl9/uVlpamSZMmad68ecrLy5PL5ZLX69Uf//Ef6+c///ll+weDQeXm5mrChAlatmyZfvGLX6Q8ZiwW07vvvqvHH39cEydO1G233aZVq1Zpz549kqQJEyYoGo2qq6tLU6ZMUV5eniTJ5XKpq6tLx44dk9PpVG5urq699toB+3c6nTp37pza2trU19cnr9d72at4sVhMH3zwgR599FG53W7dddddWrx48ZWcOgBjEMEMgNFmz57db/mjjz7Sxo0btXDhQt1+++168cUX1dXVddn+M2fOTP75mmuuUW9vb8pjnjhxQpmZmZo8eXJy3fXXXy/LsiRJzz33nD766CPdd999Kikp0U9+8hNJUnFxse6++26VlZXpnnvuUXV1tfr6+gbsf+7cudqyZYtqa2t19913a/Pmzfq///u/zxyLZVnKyMjQNddck1x33XXXpZwDgLGJYAbAaA6Ho9/yN7/5Tfl8Pr399tt655139Oijjw77MWfNmqWurq5+Ie6TTz6Rx+ORdDFY1dTUqKWlRQ8++KA2bdqkc+fOye12a9OmTfrxj3+sN954Q/v27dNbb731mcdYsWKF/vmf/1mRSESJRELbtm37zPnOmjVL3d3d+s1vfpNc197ePtxTBmAIghmAMeXMmTNKT0/X5MmT9dFHH+mHP/zhsB/D6/UqJydH27ZtUzwe1/vvv69du3YlP75j9+7dOnnypNLS0jRlyhQ5HA45HA61tLTol7/8pS5cuKApU6ZowoQJA4KWdPGq309/+lPF43FNnDhRkyZNSrb7vd/7Pf3v//6vbNuWJN1444265ZZb9J3vfEfxeFz/9m//lrxCB2D8IZgBGFO2bNmihoYG3X777aqoqNDSpUtH5DiX3ghduHChHn30UW3evFl33XWXJKm5uVmFhYWaP3++vv3tb6umpkZut1snTpzQpk2b9OUvf1lFRUVasGCBli1bNmDf8Xhcf/M3f6O77rpLixYt0qlTp/TYY49JkgoLC3X+/HndeeedybdAa2pq9M477+iuu+7S9773vUF/vhuAscNhX/pvGQAAAEYVV8wAAAAMQTADAAAwxBV9wGxzc7OeffZZXbhwQWvWrNHGjRv7bY/H4/r617+u9957TxkZGaqpqUl+MnVdXZ127typtLQ0PfXUU7rnnnskXfz6lWuvvVZpaWlyOp3atWuXJOmFF15QJBJRWlqaZsyYoa1btybfhAIAABjPUj5jlkgkFAwG9YMf/EAej0clJSXatm2bfv/3fz/Z5p/+6Z/0wQcfqKqqSuFwWPv27dMLL7ygtrY2bd68WTt37pRlWSotLVVjY6OcTqf8fr927tw54KtKenp6NGXKFEnSa6+9pra2NlVVVY3A1AEAAMyS8opZa2ursrOzk59KHQqFFIlE+gWzpqYmPfLII5Iufsp2VVWVbNtWJBJRKBSS2+2W1+tVdna2WltbNX/+/Mse71Iok6SzZ89+5qvm/38XLlxQIsE7DFfK6XRwvgxDTcxEXcxDTcxEXQZnwgTnZbelDGaWZSkrKyu57PF41NraOqDNpU/ndrlcSk9PV1dXlyzLSn5VyaW+lz45W5LWr18vh8OhtWvXau3atcn1NTU12r17t9LT0/Xaa69dwRQxOA5J/AKZhZqYibqYh5qYiboMl1H7EvM333xTHo9HnZ2dKi0t1dy5c3XHHXdIkh577DE99thjqqur0+uvv57yk70TCVvd3am/ZgUXZWRM5nwZhpqYibqYh5qYiboMzsyZ6ZfdlvKtTI/Ho46OjuSyZVkDHsb3eDzJrwjp6+vT6dOnlZmZ+bl9L/2cMWOGAoHAgKtwkrRs2TK9/fbbqYYIAAAwLqQMZrm5uYpGo4rFYorH4wqHw/L7/f3a+P1+NTQ0SJIaGxuVn58vh8Mhv9+vcDiseDyuWCymaDSqefPmqbe3Vz09PZKk3t5eHTlyRD6fT5IUjUaT+41EIpo7d+5wzRUAAMBoKW9lulwuVVRUaMOGDUokElq9erV8Pp9qa2uVk5OjgoIClZSUqLy8XIFAQNOmTVNNTY0kyefzaenSpSosLJTT6VRFRYWcTqc6Ozv18MMPS7r41mdRUZEWL14sSfrbv/1b/epXv5LD4dD111+vysrKEZw+AACAOcbFVzKdP5/g3vYg8CyAeaiJmaiLeaiJmajL4AzpGTMAAABcHQQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAzhupJGzc3NevbZZ3XhwgWtWbNGGzdu7Lc9Ho/r61//ut577z1lZGSopqZGN9xwgySprq5OO3fuVFpamp566indc889kiS/369rr71WaWlpcjqd2rVrlyTp+eef14EDBzRhwgTdeOON2rp1q6ZOnTqccwYAADBSyitmiURCVVVVqq+vVzgc1t69e9XW1tavzY4dOzR16lTt27dP69atU3V1tSSpra1N4XBY4XBY9fX1qqysVCKRSPZ79dVXtWfPnmQok6SFCxdq7969euuttzRnzhzV1dUN11wBAACMljKYtba2Kjs7W16vV263W6FQSJFIpF+bpqYmFRcXS5KCwaBaWlpk27YikYhCoZDcbre8Xq+ys7PV2tr6ucdbtGiRXK6LF/L+6I/+SB0dHV90bgAAAGNKyluZlmUpKysruezxeAaEK8uyNHv27Is7dLmUnp6urq4uWZalvLy8fn0ty0our1+/Xg6HQ2vXrtXatWsHHPtf/uVftHTp0pSTcDodysiYnLIdLnI60zhfhqEmZqIu5qEmZqIuw+eKnjEbCW+++aY8Ho86OztVWlqquXPn6o477khuf+mll+R0OrV8+fKU+0okbHV3947kcMeVjIzJnC/DUBMzURfzUBMzUZfBmTkz/bLbUt7K9Hg8/W4nWpYlj8czoE17e7skqa+vT6dPn1ZmZubn9r30c8aMGQoEAv2uwu3atUsHDx5UdXW1HA7HlcwRAABgzEsZzHJzcxWNRhWLxRSPxxUOh+X3+/u18fv9amhokCQ1NjYqPz9fDodDfr9f4XBY8XhcsVhM0WhU8+bNU29vr3p6eiRJvb29OnLkiHw+n6SLb4DW19frpZde0jXXXDPc8wUAADBWyluZLpdLFRUV2rBhgxKJhFavXi2fz6fa2lrl5OSooKBAJSUlKi8vVyAQ0LRp01RTUyNJ8vl8Wrp0qQoLC+V0OlVRUSGn06nOzk49/PDDki6+9VlUVKTFixdLkp555hnF43GVlpZKkvLy8lRVVTVS8wcAADCGw7Zte7QHMVTnzye4tz0IPAtgHmpiJupiHmpiJuoyOEN6xgwAAABXB8EMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBBXFMyam5sVDAYVCAS0ffv2Advj8bjKysoUCAS0Zs0aHT9+PLmtrq5OgUBAwWBQhw4dSq73+/1atmyZVqxYoVWrViXX//jHP1YoFNKXvvQlvfvuu0OZGwAAwJiSMpglEglVVVWpvr5e4XBYe/fuVVtbW782O3bs0NSpU7Vv3z6tW7dO1dXVkqS2tjaFw2GFw2HV19ersrJSiUQi2e/VV1/Vnj17tGvXruS6W265Rd/5znd0xx13DNccAQAAxoSUway1tVXZ2dnyer1yu90KhUKKRCL92jQ1Nam4uFiSFAwG1dLSItu2FYlEFAqF5Ha75fV6lZ2drdbW1s893s0336y5c+cOYUoAAABjkytVA8uylJWVlVz2eDwDwpVlWZo9e/bFHbpcSk9PV1dXlyzLUl5eXr++lmUll9evXy+Hw6G1a9dq7dq1X3gSTqdDGRmTv3D/3zVOZxrnyzDUxEzUxTzUxEzUZfikDGYj5c0335TH41FnZ6dKS0s1d+7cL3z7MpGw1d3dO8wjHL8yMiZzvgxDTcxEXcxDTcxEXQZn5sz0y25LeSvT4/Goo6MjuWxZljwez4A27e3tkqS+vj6dPn1amZmZn9v30s8ZM2YoEAikvMUJAAAw3qUMZrm5uYpGo4rFYorH4wqHw/L7/f3a+P1+NTQ0SJIaGxuVn58vh8Mhv9+vcDiseDyuWCymaDSqefPmqbe3Vz09PZKk3t5eHTlyRD6fbwSmBwAAMHakvJXpcrlUUVGhDRs2KJFIaPXq1fL5fKqtrVVOTo4KCgpUUlKi8vJyBQIBTZs2TTU1NZIkn8+npUuXqrCwUE6nUxUVFXI6ners7NTDDz8s6eJbn0VFRVq8eLEkad++fXrmmWd08uRJPfTQQ/rDP/xDvfzyyyN4CgAAAMzgsG3bHu1BDNX58wnubQ8CzwKYh5qYibqYh5qYiboMzpCeMQMAAMDVQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMMQVBbPm5mYFg0EFAgFt3759wPZ4PK6ysjIFAgGtWbNGx48fT26rq6tTIBBQMBjUoUOHkuv9fr+WLVumFStWaNWqVcn13d3dKi0t1ZIlS1RaWqpTp04NZX4AAABjRspglkgkVFVVpfr6eoXDYe3du1dtbW392uzYsUNTp07Vvn37tG7dOlVXV0uS2traFA6HFQ6HVV9fr8rKSiUSiWS/V199VXv27NGuXbuS67Zv364FCxbo7bff1oIFCz4zCAIAAIxHKYNZa2ursrOz5fV65Xa7FQqFFIlE+rVpampScXGxJCkYDKqlpUW2bSsSiSgUCsntdsvr9So7O1utra2fe7xIJKKVK1dKklauXKn9+/d/0bkBAACMKa5UDSzLUlZWVnLZ4/EMCFeWZWn27NkXd+hyKT09XV1dXbIsS3l5ef36WpaVXF6/fr0cDofWrl2rtWvXSpI6Ozs1a9YsSdLMmTPV2dmZchJOp0MZGZNTtsNFTmca58sw1MRM1MU81MRM1GX4pAxmI+XNN9+Ux+NRZ2enSktLNXfuXN1xxx392jgcDjkcjpT7SiRsdXf3jtRQx52MjMmcL8NQEzNRF/NQEzNRl8GZOTP9sttS3sr0eDzq6OhILluWJY/HM6BNe3u7JKmvr0+nT59WZmbm5/a99HPGjBkKBALJq3AzZszQiRMnJEknTpzQ9OnTr2iSAAAAY13KYJabm6toNKpYLKZ4PK5wOCy/39+vjd/vV0NDgySpsbFR+fn5cjgc8vv9CofDisfjisViikajmjdvnnp7e9XT0yNJ6u3t1ZEjR+Tz+ZL72r17tyRp9+7dKigoGNYJAwAAmCrlrUyXy6WKigpt2LBBiURCq1evls/nU21trXJyclRQUKCSkhKVl5crEAho2rRpqqmpkST5fD4tXbpUhYWFcjqdqqiokNPpVGdnpx5++GFJF9/6LCoq0uLFiyVJGzduVFlZmXbu3KnrrrtOL7zwwghOHwAAwBwO27bt0R7EUJ0/n+De9iDwLIB5qImZqIt5qImZqMvgDOkZMwAAAFwdBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEAQzAAAAQxDMAAAADHFFway5uVnBYFCBQEDbt28fsD0ej6usrEyBQEBr1qzR8ePHk9vq6uoUCAQUDAZ16NChfv0SiYRWrlyphx56KLmupaVFxcXFKioq0pYtW9TX1/dF5wYAADCmpAxmiURCVVVVqq+vVzgc1t69e9XW1tavzY4dOzR16lTt27dP69atU3V1tSSpra1N4XBY4XBY9fX1qqysVCKRSPZ77bXXdPPNNyeXL1y4oG984xvatm2b9u7dq+uuu04NDQ3DNVcAAACjpQxmra2tys7OltfrldvtVigUUiQS6demqalJxcXFkqRgMKiWlhbZtq1IJKJQKCS32y2v16vs7Gy1trZKkjo6OnTw4EGVlJQk99Pd3a0JEybopptukiQtXLhQb7/99rBNFgAAwGSuVA0sy1JWVlZy2ePxJMPVb7eZPXv2xR26XEpPT1dXV5csy1JeXl6/vpZlSZKee+45lZeX68yZM8ntmZmZSiQSevfdd5Wbm6t//dd/VUdHR8pJOJ0OZWRMTtkOFzmdaZwvw1ATM1EX81ATM1GX4ZMymI2EAwcOaPr06crJydHPfvaz5HqHw6Ft27Zp69atisfjWrhwodLSUj8Gl0jY6u7uHckhjysZGZM5X4ahJmaiLuahJmaiLoMzc2b6ZbelDGYej6ffVSvLsuTxeAa0aW9vV1ZWlvr6+nT69GllZmZetm9TU5OamprU3Nysc+fOqaenR0888YSqq6s1f/58vfHGG5Kkw4cPKxqNDna+AAAAY1LKy1G5ubmKRqOKxWKKx+MKh8Py+/392vj9/uRD+o2NjcrPz5fD4ZDf71c4HFY8HlcsFlM0GtW8efP0+OOPq7m5WU1NTdq2bZvy8/OTLwx0dnZKuvim5/e//339yZ/8yXDPGQAAwEgpr5i5XC5VVFRow4YNSiQSWr16tXw+n2pra5WTk6OCggKVlJSovLxcgUBA06ZNU01NjSTJ5/Np6dKlKiwslNPpVEVFhZxO5+cer76+XgcPHtSFCxd0//33a8GCBcMzUwAAAMM5bNu2R3sQQ3X+fIJ724PAswDmoSZmoi7moSZmoi6D83nPmPHJ/wAAAIYgmAEAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABiCYAYAAGAIghkAAIAhCGYAAACGIJgBAAAY4oqCWXNzs4LBoAKBgLZv3z5gezweV1lZmQKBgNasWaPjx48nt9XV1SkQCCgYDOrQoUP9+iUSCa1cuVIPPfRQcl1LS4uKi4u1YsUK3X///Tp27NgXnRsAAMCYkjKYJRIJVVVVqb6+XuFwWHv37lVbW1u/Njt27NDUqVO1b98+rVu3TtXV1ZKktrY2hcNhhcNh1dfXq7KyUolEItnvtdde080339xvX3/1V3+l6upq7dmzR0VFRXrppZeGY54AAADGSxnMWltblZ2dLa/XK7fbrVAopEgk0q9NU1OTiouLJUnBYFAtLS2ybVuRSEShUEhut1ter1fZ2dlqbW2VJHV0dOjgwYMqKSkZcMyenp7kz1mzZg15kgAAAGOBK1UDy7KUlZWVXPZ4PMlw9dttZs+efXGHLpfS09PV1dUly7KUl5fXr69lWZKk5557TuXl5Tpz5ky/fT377LPauHGjJk6cqClTpuhHP/pRykk4nQ5lZExO2Q4XOZ1pnC/DUBMzURfzUBMzUZfhkzKYjYQDBw5o+vTpysnJ0c9+9rN+21555RVt375deXl5qq+v19atW/Xss89+7v4SCVvd3b0jOeRxJSNjMufLMNTETNTFPNTETNRlcGbOTL/stpS3Mj0ejzo6OpLLlmXJ4/EMaNPe3i5J6uvr0+nTp5WZmXnZvu+8846amprk9/u1efNm/fSnP9UTTzyhkydP6he/+EXyKlthYaGOHj06uNkCAACMUSmDWW5urqLRqGKxmOLxuMLhsPx+f782fr9fDQ0NkqTGxkbl5+fL4XDI7/crHA4rHo8rFospGo1q3rx5evzxx9Xc3KympiZt27ZN+fn5qq6u1tSpU3X69Gn96le/kiQdOXJkwMsBAAAA41XKW5kul0sVFRXasGGDEomEVq9eLZ/Pp9raWuXk5KigoEAlJSUqLy9XIBDQtGnTVFNTI0ny+XxaunSpCgsL5XQ6VVFRIafT+bnH+ta3vqVHH31UDodD06ZN03PPPTd8swUAADCYw7Zte7QHMVTnzye4tz0IPAtgHmpiJupiHmpiJuoyOEN6xgwAAABXB8EMAADAEAQzAAAAQxDMAAAADEEwAwAAMATBDAAAwBAEMwAAAEMQzAAAAAxBMAMAADAEwQwAAMAQBDMAAABDEMwAAAAMQTADAAAwBMEMAADAEA7btu3RHgQAAAC4YgYAAGAMghkAAIAhCGYAAACGIJgBAAAYgmAGAABgCIIZAACAIQhmAAAAhiCYjVPd3d0qLS3VkiVLVFpaqlOnTn1mu4aGBi1ZskRLlixRQ0PDgO1f+9rXVFRUNNLD/Z0wlJqcPXtWGzdu1H333adQKKTq6uqrOfRxqbm5WcFgUIFAQNu3bx+wPR6Pq6ysTIFAQGvWrNHx48eT2+rq6hQIBBQMBnXo0KGrOexx7YvW5MiRI1q1apWWLVumVatWqaWl5WoPfdwayu+JJH3yySeaP3++Xn755as15LHPxrj0/PPP23V1dbZt23ZdXZ397W9/e0Cbrq4u2+/3211dXXZ3d7ft9/vt7u7u5PbGxkZ78+bNdigUumrjHs+GUpPe3l67paXFtm3bPnfunH3//ffbBw8evKrjH0/6+vrsgoIC++OPP7bPnTtnL1u2zP7www/7tXn99dftp59+2rZt2967d6/9F3/xF7Zt2/aHH35oL1u2zD537pz98ccf2wUFBXZfX99Vn8N4M5SavPfee3ZHR4dt27b9wQcf2IsWLbq6gx+nhlKTSzZt2mRv2rTJrq+vv2rjHuu4YjZORSIRrVy5UpK0cuVK7d+/f0Cbw4cPa+HChcrIyNC0adO0cOHC5P/+z5w5ox/84Af68z//86s67vFsKDW55pprlJ+fL0lyu9269dZbZVnWVR3/eNLa2qrs7Gx5vV653W6FQiFFIpF+bZqamlRcXCxJCgaDamlpkW3bikQiCoVCcrvd8nq9ys7OVmtr62hMY1wZSk1uvfVWeTweSZLP59O5c+cUj8ev+hzGm6HURJL279+v66+/Xj6f76qPfSwjmI1TnZ2dmjVrliRp5syZ6uzsHNDGsixlZWUllz0eT/If+9raWj344IOaNGnS1Rnw74Ch1uSSTz/9VAcOHNCCBQtGdsDj2JWcZ8uyNHv2bEmSy+VSenq6urq6rqgvBm8oNfltjY2NuvXWW+V2u0d+0OPcUGpy5swZff/739cjjzxyVcc8HrhGewD44tatW6df//rXA9aXlZX1W3Y4HHI4HFe83/fff18ff/yxnnzyyQHPC+DzjVRNLunr69PmzZv11a9+VV6v9wuPExiPPvzwQ1VXV+sf/uEfRnsov/P+7u/+Tg888BwyXkQAAAKcSURBVICuvfba0R7KmEMwG8NeeeWVy26bMWOGTpw4oVmzZunEiROaPn36gDYej0c///nPk8uWZenOO+/U0aNH9V//9V/y+/3q6+vTyZMn9dWvflX/+I//OBLTGFdGqiaXPP3005ozZ47WrVs3nMP+nePxeNTR0ZFctiwreSvst9u0t7crKytLfX19On36tDIzM6+oLwZvKDWRpI6ODj3yyCN6/vnndeONN17VsY9XQ6nJf/7nf6qxsVHV1dX69NNPlZaWpokTJ+orX/nK1Z7GmMOtzHHK7/dr9+7dkqTdu3eroKBgQJtFixbp8OHDOnXqlE6dOqXDhw9r0aJF+tM//VMdPnxYTU1NeuONNzRnzhxC2TAYSk0kqaamRj09PXryySev6rjHo9zcXEWjUcViMcXjcYXDYfn9/n5t/H5/8q3YxsZG5efny+FwyO/3KxwOKx6PKxaLKRqNat68eaMxjXFlKDX59NNPtXHjRj3++OP68pe/PBrDH5eGUpM33nhDTU1Nampq0gMPPKCHHnqIUHalRvnlA4yQkydP2n/2Z39mBwIB+4EHHrC7urps27bt1tZW+8knn0y227Fjh33vvffa9957r71z584B+4nFYryVOUyGUpP29nb7lltuse+77z57+fLl9vLly+0f/ehHozKP8eLgwYP2kiVL7IKCAvu73/2ubdu2/cILL9j79++3bdu2f/Ob39ibNm2y7733Xnv16tX2xx9/nOz73e9+1y4oKLCXLFnC27HD6IvW5O///u/tvLy85O/G8uXL7V//+tejNo/xZCi/J5e8+OKLvJU5CA7b/v9enwAAAMCo4lYmAACAIQhmAAAAhiCYAQAAGIJgBgAAYAiCGQAAgCEIZgAAAIYgmAEAABji/wHWUarjVEZu+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAEICAYAAADiGKj0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df3TU1Z3/8ecwcRRCfmHJhN1N4yJB1/IrhZpYKciEYRJGEiBJT5sjVVqq7aJuVqFS1KxQ1KNlG6DqMawtRV30mJYf1sGGkCgJEn9tq0M9rEh7YsIx+aSaQKiYTGa83z9S59tsxARD4EN8Pf7yk3vnfu6974PndT73M4nDGGMQEREREdsaca4nICIiIiKfTYFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ETElo4ePcpll11GOBwGYNmyZezYsWNAfU/Xo48+yp133vm553oq27dv59vf/vYZH3coLFmyhIqKinM9DRE5BQU2ERkS3/ve99i4cWOfn+/du5err776tMPVY489xqJFiwY9r1deeYVZs2b1+tkPfvAD7r333kGPbVcKYyLnPwU2ERkSixYt4tlnn+X//m7uZ599lgULFhATE3OOZiYicv5RYBORITF37lyOHTvG66+/Hv3Z8ePHeeGFF1i4cCEAL774IgsXLuSrX/0qs2fP5uc///kpx/v7p0SRSIQHHniAzMxMsrOz2bdvX6++v/nNb8jNzSUjI4Ps7GyefvppAE6ePMn3v/99WltbycjIICMjA8uy+PnPf86KFSuin6+ursbv9zNjxgyWLFnCn/70p2ibx+PhF7/4BQsWLGD69OmUlJTQ1dU1oD35/e9/T0FBAdOnT6egoIDf//730bbt27eTnZ1NRkYGHo+HZ599FoB3332X6667junTp5OZmUlJScmnjt3V1cWKFSvIzMxkxowZFBQU8P7771NWVsbrr7/O2rVrycjIYO3atQC89NJL5OTkMH36dNauXdsnWIuIzRgRkSFy5513mtWrV0evn3rqKZOXlxe9fvnll83//u//mkgkYg4dOmSuuuoqU1VVZYwxpqmpyUycONF0d3cbY4y57rrrzDPPPGOMMWbbtm3G5/OZ9957z7S3t5vrrruuV98XXnjBvPvuu+bjjz82r7zyipkyZYr54x//GL3nN77xjV7z3LRpk7n99tuNMcb8+c9/NlOnTjX79+83oVDIbN682cydO9d0dXUZY4yZM2eOKSgoMC0tLaa9vd3k5OSYbdu2fer6f/Ob35hvfetbxhhj2tvbzYwZM8yOHTtMd3e3+e1vf2tmzJhh2trazIcffmgyMjLMn/70J2OMMZZlmcOHDxtjjPn3f/9388gjj5hIJGI6OzvNa6+99qn3euqpp8xNN91kTp48acLhsDl48KA5ceJEn70zxpgPPvjATJs2zTz//PMmFAqZLVu2mH/5l3/p1UdE7EVP2ERkyCxcuJDKysroE6idO3f2eg8tMzOTyy67jBEjRnD55Zfj9/t59dVX+x33+eef5/rrr2fcuHEkJiZy00039Wq/5ppr+PKXv4zD4eDKK6/k6quv7vWk77Ps3r2b2bNnc/XVV3PBBRfwve99j87OTv7whz9E+yxZsgS3201iYiJz5szh0KFD/Y774osvkpaWxsKFC4mJieHaa69l/PjxvPDCCwCMGDGCd955h87OTpKTk0lPTwcgJiaG9957j9bWVi688EJmzJjxqePHxMRw7Ngx3n33XZxOJ5MmTWL06NGf2re2tpb09HRycnK44IILuP766/nSl740oP0RkXNDgU1EhsyMGTNISkpi7969NDY2cvDgQa699tpo+5tvvsmSJUvIyspi+vTpPP3007S3t/c7bmtrK+PGjYte/8M//EOv9n379vHNb36TK6+8khkzZlBbWzugcT8Z++/HGzFiBOPGjcOyrOjPxo4dG/3vkSNHcvLkydMe95N5W5bFqFGjKCsr4+mnn2bmzJnceOON0WPYlStXYoyhsLAQv9/Pr3/9608dPz8/n5kzZ3Lbbbcxc+ZMHnzwQbq7u085l5SUlOi1w+HotZ8iYj8KbCIypPLz89m5cyfPPvssM2fO7PUk5/bbb4++g/Y///M/fOtb3xrQu1Rjx46lubk5ev33/x0Khbj11lv57ne/y0svvcTrr7/OrFmzouM6HI7PHDs5OZn33nsvem2Mobm5GbfbPeA1D2TcT+b9ybjf+MY32LJlC/v372f8+PHcfffd0bWuW7eO/fv3s2bNGtasWcO7777bZ/wLLriAm2++md27d/P000/z4osvsnPnzk+dy9ixY2lpaemzRhGxLwU2ERlSCxcupL6+nmeeeSb6ZYNPfPjhhyQkJHDhhRcSDAZ57rnnBjRmbm4uTzzxBC0tLRw/fpzNmzdH20KhEKFQiDFjxhATE8O+fft46aWXou0XX3wxx44d48SJE6cce9++fdTX19Pd3c0vf/lLXC4XGRkZn2P1/9/s2bNpaGjgt7/9LeFwmN27d3PkyBGuueYa3n//ffbu3cvJkydxuVyMGjWKESN6/vf8/PPPR8NVQkICDocj2vb3Xn75Zd5++20ikQijR48mJiYm2u9LX/oSTU1NvebyzjvvsGfPHsLhMI8//jjvv//+oNYnIkNLgU1EhtQ//dM/kZGRwUcffUR2dnavtv/4j/9g06ZNZGRk8PDDD5ObmzugMb/5zW8yc+ZM8vPzWbRoEfPmzYu2jR49mrvuuouSkhK+9rWv8dxzz+HxeKLtl156KX6/n7lz5zJjxoxeR50A48eP56c//Sk/+clPyMrK4oUXXuDRRx/F5XINYhcgKSmJRx99lC1btpCZmcljjz3Go48+ypgxY/j444/51a9+xTe+8Q2uvPJKXnvtNe655x4ADh48SFFRERkZGfzwhz/kzjvvJDU1tc/477//PrfeeivTp09n/vz5XHnlleTn5wPwne98h8rKSr72ta+xbt06xowZw8aNG/nP//xPMjMzeffdd/nqV786qPWJyNBymIGcP4iIiIjIOaMnbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnPD+q8vf/zxx0Qi+k7FQDmdDu2XDaku9qOa2JPqYj+qyem54ALnKduGdWCLRAzHjvX/G8ilR2LiKO2XDaku9qOa2JPqYj+qyekZOzbulG06EhURERGxOQU2EREREZtTYBMRERGxuQEFttraWnw+H16vt9ff7PtEKBSipKQEr9dLUVERR48ejbaVl5fj9Xrx+XzU1dUB0NXVRWFhIXl5efj9fjZt2hTtX1xcTH5+Pvn5+cycOZN//dd/BXr+OPG6devwer0sWLCAt956a1ALFxERETlf9Pulg0gkwtq1a9myZQtut5vCwkI8Hg8TJkyI9qmoqCA+Pp6qqioCgQDr169nw4YNHDlyhEAgQCAQwLIsli5dSmVlJS6Xi61btxIbG0t3dzfFxcXMmjWLadOmsW3btui4t9xyS/RvD9bW1tLQ0MCePXt48803ueeee6ioqBiCLRERERGxl36fsAWDQdLS0khNTcXlcuH3+6muru7Vp6amhkWLFgHg8/mor6/HGEN1dTV+vx+Xy0VqaippaWkEg0EcDgexsbEAhMNhwuEwDoej15h//etfefnll5k7dy4A1dXVLFy4EIfDwbRp0+jo6KC1tfWMbIKIiIiInfX7hM2yLFJSUqLXbrebYDDYp8+4ceN6BoyJIS4ujvb2dizLYurUqb0+a1kW0PPkbvHixTQ2NlJcXNyrH8DevXu56qqrGD169KfOIyUlBcuySE5OPuXcnU4HiYmj+lui/I3TOUL7ZUOqi/2oJvakutiPanLmnLPfw+Z0Otm1axcdHR0sX76cw4cPM3HixGj7c889R1FR0aDuod/Ddnr0+3LsSXWxH9XEnlQX+1FNTs+gfg+b2+2mpaUlem1ZFm63u0+f5uZmoOeI88SJEyQlJQ3os/Hx8WRmZka/kADQ1tbGwYMHueaaa045j5aWlj5jiYiIiAxH/Qa2yZMn09DQQFNTE6FQiEAggMfj6dXH4/GwY8cOACorK8nKysLhcODxeAgEAoRCIZqammhoaGDKlCm0tbXR0dEBQGdnJwcOHGD8+PHR8SorK7nmmmu48MILe91j586dGGN44403iIuL+8zjUBEREZHhot8j0ZiYGEpLS1m2bBmRSISCggLS09PZuHEjkyZNIjs7m8LCQlauXInX6yUhIYGysjIA0tPTyc3NZf78+TidTkpLS3E6nbS2trJq1SoikQjGGHJycpgzZ070nrt37+b73/9+r3nMnj2bffv24fV6GTlyJPfdd98Z3goRERERe3IYY4btX2Xt7o7o7Pw06F0De1Jd7Ec1sSfVxX5Uk9OjvyUqIiIich5TYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZsbUGCrra3F5/Ph9XrZvHlzn/ZQKERJSQler5eioiKOHj0abSsvL8fr9eLz+airqwOgq6uLwsJC8vLy8Pv9bNq0KdrfGENZWRk+n4/c3Fwef/xxAF555RWmT59Ofn4++fn5PPTQQ4NauIiIiMj5Iqa/DpFIhLVr17JlyxbcbjeFhYV4PB4mTJgQ7VNRUUF8fDxVVVUEAgHWr1/Phg0bOHLkCIFAgEAggGVZLF26lMrKSlwuF1u3biU2Npbu7m6Ki4uZNWsW06ZNY/v27TQ3N/P8888zYsQIPvjgg+h9ZsyYQXl5+dDshIiIiIhN9fuELRgMkpaWRmpqKi6XC7/fT3V1da8+NTU1LFq0CACfz0d9fT3GGKqrq/H7/bhcLlJTU0lLSyMYDOJwOIiNjQUgHA4TDodxOBwAPPXUUyxfvpwRI3qmdvHFF5/RBYuIiIicb/p9wmZZFikpKdFrt9tNMBjs02fcuHE9A8bEEBcXR3t7O5ZlMXXq1F6ftSwL6Hlyt3jxYhobGykuLo72a2pqYvfu3VRVVTFmzBjuuusuLrnkEgDeeOMN8vLySE5O5o477iA9Pf0z5+50OkhMHDWAbRAAp3OE9suGVBf7UU3sSXWxH9XkzOk3sA0Vp9PJrl276OjoYPny5Rw+fJiJEycSCoW48MIL2b59O3v27GH16tVs27aNr3zlK9TU1BAbG8u+fftYvnw5e/bs+cx7RCKGY8dOnqUVnf8SE0dpv2xIdbEf1cSeVBf7UU1Oz9ixcads6/dI1O1209LSEr22LAu3292nT3NzM9BzxHnixAmSkpIG9Nn4+HgyMzOjX0hwu914vV4AvF4vb7/9NgCjR4+OHqPOnj2bcDhMW1tbf9MXEREROe/1G9gmT55MQ0MDTU1NhEIhAoEAHo+nVx+Px8OOHTsAqKysJCsrC4fDgcfjIRAIEAqFaGpqoqGhgSlTptDW1kZHRwcAnZ2dHDhwgPHjxwMwd+5cXnnlFQBeffXV6HHoX/7yF4wxQM97dR9//DFJSUlnZhdEREREbKzfI9GYmBhKS0tZtmwZkUiEgoIC0tPT2bhxI5MmTSI7O5vCwkJWrlyJ1+slISGBsrIyANLT08nNzWX+/Pk4nU5KS0txOp20trayatUqIpEIxhhycnKYM2cOADfeeCMrVqxg69atjBo1invvvRfoCYJPPfUUTqeTiy66iJ/97GfRLyqIiIiIDGcO88ljq2Gouzuis/PToHcN7El1sR/VxJ5UF/tRTU7PoN5hExEREZFzS4FNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsToFNRERExOYU2ERERERsbkCBrba2Fp/Ph9frZfPmzX3aQ6EQJSUleL1eioqKOHr0aLStvLwcr9eLz+ejrq4OgK6uLgoLC8nLy8Pv97Np06Zof2MMZWVl+Hw+cnNzefzxx6M/X7duHV6vlwULFvDWW28NauEiIiIi54uY/jpEIhHWrl3Lli1bcLvdFBYW4vF4mDBhQrRPRUUF8fHxVFVVEQgEWL9+PRs2bODIkSMEAgECgQCWZbF06VIqKytxuVxs3bqV2NhYuru7KS4uZtasWUybNo3t27fT3NzM888/z4gRI/jggw+AntDY0NDAnj17ePPNN7nnnnuoqKgYup0RERERsYl+n7AFg0HS0tJITU3F5XLh9/uprq7u1aempoZFixYB4PP5qK+vxxhDdXU1fr8fl8tFamoqaWlpBINBHA4HsbGxAITDYcLhMA6HA4CnnnqK5cuXM2JEz9QuvvhiAKqrq1m4cCEOh4Np06bR0dFBa2vrmdsJEREREZvq9wmbZVmkpKREr91uN8FgsE+fcePG9QwYE0NcXBzt7e1YlsXUqVN7fdayLKDnyd3ixYtpbGykuLg42q+pqYndu3dTVVXFmDFjuOuuu7jkkkv6zCMlJQXLskhOTj7l3J1OB4mJowayDwI4nSO0XzakutiPamJPqov9qCZnTr+Bbag4nU527dpFR0cHy5cv5/Dhw0ycOJFQKMSFF17I9u3b2bNnD6tXr2bbtm2f6x6RiOHYsZNneObDV2LiKO2XDaku9qOa2JPqYj+qyekZOzbulG39Hom63W5aWlqi15Zl4Xa7+/Rpbm4Geo44T5w4QVJS0oA+Gx8fT2ZmZvQLCW63G6/XC4DX6+Xtt9/+1Hm0tLT0GUtERERkOOo3sE2ePJmGhgaampoIhUIEAgE8Hk+vPh6Phx07dgBQWVlJVlYWDocDj8dDIBAgFArR1NREQ0MDU6ZMoa2tjY6ODgA6Ozs5cOAA48ePB2Du3Lm88sorALz66qtccskl0Xvs3LkTYwxvvPEGcXFxn3kcKiIiIjJc9HskGhMTQ2lpKcuWLSMSiVBQUEB6ejobN25k0qRJZGdnU1hYyMqVK/F6vSQkJFBWVgZAeno6ubm5zJ8/H6fTSWlpKU6nk9bWVlatWkUkEsEYQ05ODnPmzAHgxhtvZMWKFWzdupVRo0Zx7733AjB79mz27duH1+tl5MiR3HfffUO4LSIiIiL24TDGmHM9iaHS3R3R2flp0LsG9qS62I9qYk+qi/2oJqdnUO+wiYiIiMi5pcAmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2N6DAVltbi8/nw+v1snnz5j7toVCIkpISvF4vRUVFHD16NNpWXl6O1+vF5/NRV1cHQFdXF4WFheTl5eH3+9m0aVO0/6pVq/B4POTn55Ofn8+hQ4cAeOWVV5g+fXr05w899NCgFi4iIiJyvojpr0MkEmHt2rVs2bIFt9tNYWEhHo+HCRMmRPtUVFQQHx9PVVUVgUCA9evXs2HDBo4cOUIgECAQCGBZFkuXLqWyshKXy8XWrVuJjY2lu7ub4uJiZs2axbRp0wD40Y9+RE5OTp+5zJgxg/Ly8jO4fBERERH76/cJWzAYJC0tjdTUVFwuF36/n+rq6l59ampqWLRoEQA+n4/6+nqMMVRXV+P3+3G5XKSmppKWlkYwGMThcBAbGwtAOBwmHA7jcDiGYHkiIiIi579+n7BZlkVKSkr02u12EwwG+/QZN25cz4AxMcTFxdHe3o5lWUydOrXXZy3LAnqe3C1evJjGxkaKi4t79SsrK+Phhx/mqquuYsWKFbhcLgDeeOMN8vLySE5O5o477iA9Pf0z5+50OkhMHNXfEuVvnM4R2i8bUl3sRzWxJ9XFflSTM6ffwDZUnE4nu3btoqOjg+XLl3P48GEmTpzIbbfdxtixY+nu7ubuu+9m8+bN3HzzzXzlK1+hpqaG2NhY9u3bx/Lly9mzZ89n3iMSMRw7dvIsrej8l5g4SvtlQ6qL/agm9qS62I9qcnrGjo07ZVu/R6Jut5uWlpbotWVZuN3uPn2am5uBniPOEydOkJSUNKDPxsfHk5mZGf1CQnJyMg6HA5fLxeLFizl48CAAo0ePjh6jzp49m3A4TFtbW3/TFxERETnv9RvYJk+eTENDA01NTYRCIQKBAB6Pp1cfj8fDjh07AKisrCQrKwuHw4HH4yEQCBAKhWhqaqKhoYEpU6bQ1tZGR0cHAJ2dnRw4cIDx48cD0NraCoAxhr1790aPPf/yl79gjAF63qv7+OOPSUpKOkPbICIiImJf/R6JxsTEUFpayrJly4hEIhQUFJCens7GjRuZNGkS2dnZFBYWsnLlSrxeLwkJCZSVlQGQnp5Obm4u8+fPx+l0UlpaitPppLW1lVWrVhGJRDDGkJOTw5w5cwBYsWIF7e3tGGO4/PLLWbNmDdATBJ966imcTicXXXQRP/vZz/RFBREREflCcJhPHlsNQ93dEZ2dnwa9a2BPqov9qCb2pLrYj2pyegb1DpuIiIiInFsKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMKbCIiIiI2p8AmIiIiYnMDCmy1tbX4fD68Xi+bN2/u0x4KhSgpKcHr9VJUVMTRo0ejbeXl5Xi9Xnw+H3V1dQB0dXVRWFhIXl4efr+fTZs2RfuvWrUKj8dDfn4++fn5HDp0CABjDOvWrcPr9bJgwQLeeuutQS1cRERE5HwR01+HSCTC2rVr2bJlC263m8LCQjweDxMmTIj2qaioID4+nqqqKgKBAOvXr2fDhg0cOXKEQCBAIBDAsiyWLl1KZWUlLpeLrVu3EhsbS3d3N8XFxcyaNYtp06YB8KMf/YicnJxe86itraWhoYE9e/bw5ptvcs8991BRUXGGt0NERETEfvp9whYMBklLSyM1NRWXy4Xf76e6urpXn5qaGhYtWgSAz+ejvr4eYwzV1dX4/X5cLhepqamkpaURDAZxOBzExsYCEA6HCYfDOByOz5xHdXU1CxcuxOFwMG3aNDo6Omhtbf286xYRERE5b/T7hM2yLFJSUqLXbrebYDDYp8+4ceN6BoyJIS4ujvb2dizLYurUqb0+a1kW0PPkbvHixTQ2NlJcXNyrX1lZGQ8//DBXXXUVK1aswOVy9ZlHSkoKlmWRnJx8yrk7nQ4SE0f1t0T5G6dzhPbLhlQX+1FN7El1sR/V5MzpN7ANFafTya5du+jo6GD58uUcPnyYiRMncttttzF27Fi6u7u5++672bx5MzfffPPnukckYjh27OQZnvnwlZg4SvtlQ6qL/agm9qS62I9qcnrGjo07ZVu/R6Jut5uWlpbotWVZuN3uPn2am5uBniPOEydOkJSUNKDPxsfHk5mZGf1CQnJyMg6HA5fLxeLFizl48OCnzqOlpaXPWCIiIiLDUb+BbfLkyTQ0NNDU1EQoFCIQCODxeHr18Xg87NixA4DKykqysrJwOBx4PB4CgQChUIimpiYaGhqYMmUKbW1tdHR0ANDZ2cmBAwcYP348QPS9NGMMe/fuJT09PXqPnTt3YozhjTfeIC4u7jOPQ0VERESGi36PRGNiYigtLWXZsmVEIhEKCgpIT09n48aNTJo0iezsbAoLC1m5ciVer5eEhATKysoASE9PJzc3l/nz5+N0OiktLcXpdNLa2sqqVauIRCIYY8jJyWHOnDkArFixgvb2dowxXH755axZswaA2bNns2/fPrxeLyNHjuS+++4bwm0RERERsQ+HMcac60kMle7uiM7OT4PeNbAn1cV+VBN7Ul3sRzU5PYN6h01EREREzi0FNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsTkFNhERERGbU2ATERERsbkBBbba2lp8Ph9er5fNmzf3aQ+FQpSUlOD1eikqKuLo0aPRtvLycrxeLz6fj7q6OgC6urooLCwkLy8Pv9/Ppk2b+oy5bt06MjIyotfbt28nKyuL/Px88vPzqaioOO3FioiIiJyPYvrrEIlEWLt2LVu2bMHtdlNYWIjH42HChAnRPhUVFcTHx1NVVUUgEGD9+vVs2LCBI0eOEAgECAQCWJbF0qVLqaysxOVysXXrVmJjY+nu7qa4uJhZs2Yxbdo0AA4ePMjx48f7zGX+/PmUlpaeweWLiIiI2F+/T9iCwSBpaWmkpqbicrnw+/1UV1f36lNTU8OiRYsA8Pl81NfXY4yhuroav9+Py+UiNTWVtLQ0gsEgDoeD2NhYAMLhMOFwGIfDAfQExAcffJCVK1ee6bWKiIiInJf6fcJmWRYpKSnRa7fbTTAY7NNn3LhxPQPGxBAXF0d7ezuWZTF16tRen7UsC+gJZosXL6axsZHi4uJovyeffJLs7GySk5P7zGXPnj289tpr/PM//zM//vGPo/c8FafTQWLiqP6WKH/jdI7QftmQ6mI/qok9qS72o5qcOf0GtqHidDrZtWsXHR0dLF++nMOHD5OQkMDvfvc7nnjiiT7958yZw7XXXovL5eLpp5/mjjvu4PHHH//Me0QihmPHTg7VEoadxMRR2i8bUl3sRzWxJ9XFflST0zN2bNwp2/o9EnW73bS0tESvLcvC7Xb36dPc3Az0HHGeOHGCpKSkAX02Pj6ezMxM6urqOHToEI2NjcybNw+Px8NHH32E1+sFICkpCZfLBUBRURFvvfVWf1MXERERGRb6DWyTJ3HVCH8AAA7mSURBVE+moaGBpqYmQqEQgUAAj8fTq4/H42HHjh0AVFZWkpWVhcPhwOPxEAgECIVCNDU10dDQwJQpU2hra6OjowOAzs5ODhw4wPjx47nmmmt46aWXqKmpoaamhpEjR1JVVQVAa2tr9H41NTVceumlZ2wTREREROys3yPRmJgYSktLWbZsGZFIhIKCAtLT09m4cSOTJk0iOzubwsJCVq5cidfrJSEhgbKyMgDS09PJzc1l/vz5OJ1OSktLcTqdtLa2smrVKiKRCMYYcnJymDNnzmfO44knnqCmpgan00lCQgL333//mdkBEREREZtzGGPMuZ7EUOnujujs/DToXQN7Ul3sRzWxJ9XFflST0zOod9hERERE5NxSYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZsbUGCrra3F5/Ph9XrZvHlzn/ZQKERJSQler5eioiKOHj0abSsvL8fr9eLz+airqwOgq6uLwsJC8vLy8Pv9bNq0qc+Y69atIyMjY0D3EBERERnO+g1skUiEtWvX8thjjxEIBHjuuec4cuRIrz4VFRXEx8dTVVXFDTfcwPr16wE4cuQIgUCAQCDAY489xpo1a4hEIrhcLrZu3cqzzz7Lzp07qaur44033oiOd/DgQY4fPz6ge4iIiIgMd/0GtmAwSFpaGqmpqbhcLvx+P9XV1b361NTUsGjRIgB8Ph/19fUYY6iursbv9+NyuUhNTSUtLY1gMIjD4SA2NhaAcDhMOBzG4XAAPQHxwQcfZOXKlQO6h4iIiMhwF9NfB8uySElJiV673W6CwWCfPuPGjesZMCaGuLg42tvbsSyLqVOn9vqsZVlATzBbvHgxjY2NFBcXR/s9+eSTZGdnk5ycPKB7jBkz5pRzdzodJCaO6m+J8jdO5wjtlw2pLvajmtiT6mI/qsmZ029gGypOp5Ndu3bR0dHB8uXLOXz4MAkJCfzud7/jiSeeOCP3iEQMx46dPCNjfREkJo7SftmQ6mI/qok9qS72o5qcnrFj407Z1u+RqNvtpqWlJXptWRZut7tPn+bmZqDniPPEiRMkJSUN6LPx8fFkZmZSV1fHoUOHaGxsZN68eXg8Hj766CO8Xu9n3kNERERkuOs3sE2ePJmGhgaampoIhUIEAgE8Hk+vPh6Phx07dgBQWVlJVlYWDocDj8dDIBAgFArR1NREQ0MDU6ZMoa2tjY6ODgA6Ozs5cOAA48eP55prruGll16ipqaGmpoaRo4cSVVV1WfeQ0RERGS46/dINCYmhtLSUpYtW0YkEqGgoID09HQ2btzIpEmTyM7OprCwkJUrV+L1eklISKCsrAyA9PR0cnNzmT9/Pk6nk9LSUpxOJ62traxatYpIJIIxhpycHObMmfOZ8zjVPURERESGO4cZxl+17O6O6Oz8NOhdA3tSXexHNbEn1cV+VJPTM6h32ERERETk3FJgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERm1NgExEREbE5BTYRERERmxtQYKutrcXn8+H1etm8eXOf9lAoRElJCV6vl6KiIo4ePRptKy8vx+v14vP5qKurA6Crq4vCwkLy8vLw+/1s2rQp2n/16tXk5eWxYMECbr31Vj788EMAtm/fTlZWFvn5+eTn51NRUTGohYuIiIicL2L66xCJRFi7di1btmzB7XZTWFiIx+NhwoQJ0T4VFRXEx8dTVVVFIBBg/fr1bNiwgSNHjhAIBAgEAliWxdKlS6msrMTlcrF161ZiY2Pp7u6muLiYWbNmMW3aNFavXs3o0aMBuP/++/nv//5vbrzxRgDmz59PaWnpEG2FiIiIiD31+4QtGAySlpZGamoqLpcLv99PdXV1rz41NTUsWrQIAJ/PR319PcYYqqur8fv9uFwuUlNTSUtLIxgM4nA4iI2NBSAcDhMOh3E4HADRsGaMobOz84wuVkREROR81G9gsyyLlJSU6LXb7cayrD59xo0bB0BMTAxxcXG0t7d/5mcjkQj5+fl8/etf5+tf/zpTp06N9vvxj3/M1VdfzZ///GeWLFkS/fmePXuiR6XNzc2fc8kiIiIi55d+j0SHitPpZNeuXXR0dLB8+XIOHz7MxIkTgZ6j0Egkwk9+8hN2795NQUEBc+bM4dprr8XlcvH0009zxx138Pjjj/dzDweJiaPOxnKGBadzhPbLhlQX+1FN7El1sR/V5MzpN7C53W5aWlqi15Zl4Xa7+/Rpbm4mJSWFcDjMiRMnSEpKGtBn4+PjyczMpK6uLhrYoCfQ+f1+HnvsMQoKCkhKSoq2FRUV8dOf/rTfxUUihmPHTvbbT3okJo7SftmQ6mI/qok9qS72o5qcnrFj407Z1u+R6OTJk2loaKCpqYlQKEQgEMDj8fTq4/F42LFjBwCVlZVkZWXhcDjweDwEAgFCoRBNTU00NDQwZcoU2tra6OjoAKCzs5MDBw4wfvx4jDG8++67QM87bDU1NYwfPx6A1tbW6P1qamq49NJLT3MbRERERM5P/T5hi4mJobS0lGXLlhGJRCgoKCA9PZ2NGzcyadIksrOzKSwsZOXKlXi9XhISEigrKwMgPT2d3Nxc5s+fj9PppLS0FKfTSWtrK6tWrSISiWCMIScnhzlz5vDxxx9zxx138OGHH2KM4bLLLmPNmjUAPPHEE9TU1OB0OklISOD+++8f2p0RERERsQmHMcac60kMle7uiB7FngY9urYn1cV+VBN7Ul3sRzU5PYM6EhURERGRc0uBTURERMTmFNhEREREbE6BTURERMTmFNhEREREbE6BTURERMTmFNhEREREbE6BTURERMTmhvUvzhUREREZDvSETURERMTmFNhEREREbE6BTURERMTmFNhEREREbE6BTURERMTmFNhEREREbE6BTURERMTmFNi+YI4dO8bSpUuZN28eS5cu5fjx45/ab8eOHcybN4958+axY8eOPu0/+MEPuPbaa4d6ul8Yg6nLRx99xI033khOTg5+v5/169efzakPO7W1tfh8PrxeL5s3b+7THgqFKCkpwev1UlRUxNGjR6Nt5eXleL1efD4fdXV1Z3Paw9rnrclLL73E4sWLWbBgAYsXL6a+vv5sT31YG8y/FYD33nuPjIwMfvGLX5ytKZ/fjHyhPPDAA6a8vNwYY0x5ebl58MEH+/Rpb283Ho/HtLe3m2PHjhmPx2OOHTsWba+srDS33Xab8fv9Z23ew91g6nLy5ElTX19vjDGmq6vLfPvb3zYvvvjiWZ3/cBEOh012drZpbGw0XV1dZsGCBeadd97p1efJJ580d999tzHGmOeee87827/9mzHGmHfeeccsWLDAdHV1mcbGRpOdnW3C4fBZX8NwM5iavPXWW6alpcUYY8zbb79tZs6ceXYnP4wNpi6fuOWWW8wtt9xiHnvssbM27/OZnrB9wVRXV7Nw4UIAFi5cyN69e/v02b9/P1dffTWJiYkkJCRw9dVXR58WfPjhh2zZsoUf/vCHZ3Xew91g6jJy5EiysrIAcLlcXHHFFViWdVbnP1wEg0HS0tJITU3F5XLh9/uprq7u1aempoZFixYB4PP5qK+vxxhDdXU1fr8fl8tFamoqaWlpBIPBc7GMYWUwNbniiitwu90ApKen09XVRSgUOutrGI4GUxeAvXv38o//+I+kp6ef9bmfrxTYvmA++OADkpOTARg7diwffPBBnz6WZZGSkhK9drvd0QCwceNGvvvd73LRRRednQl/QQy2Lp/o6OjghRde4KqrrhraCQ9TA9ljy7IYN24cADExMcTFxdHe3j6gz8rpG0xN/l5lZSVXXHEFLpdr6Cf9BTCYunz44Yf813/9FzfffPNZnfP5LuZcT0DOvBtuuIH333+/z89LSkp6XTscDhwOx4DHPXToEI2NjaxevbrPuwjSv6GqyyfC4TC33XYbS5YsITU19XPPU2S4eeedd1i/fj2//OUvz/VUBHjooYe4/vrriY2NPddTOa8osA1Dv/rVr07ZdvHFF9Pa2kpycjKtra2MGTOmTx+3282rr74avbYsiyuvvJI//OEP/PGPf8Tj8RAOh2lra2PJkiU88cQTQ7GMYWeo6vKJu+++m0suuYQbbrjhTE77C8XtdtPS0hK9tiwreqT2932am5tJSUkhHA5z4sQJkpKSBvRZOX2DqQlAS0sLN998Mw888ABf/vKXz+rch7PB1OXNN9+ksrKS9evX09HRwYgRI7jwwgu57rrrzvYyzis6Ev2C8Xg87Ny5E4CdO3eSnZ3dp8/MmTPZv38/x48f5/jx4+zfv5+ZM2dSXFzM/v37qampYdu2bVxyySUKa2fIYOoCUFZWxl//+ldWr159Vuc93EyePJmGhgaampoIhUIEAgE8Hk+vPh6PJ/oN3crKSrKysnA4HHg8HgKBAKFQiKamJhoaGpgyZcq5WMawMpiadHR0cOONN3L77bczffr0czH9YWswddm2bRs1NTXU1NRw/fXXc9NNNymsDcQ5/tKDnGVtbW3mO9/5jvF6veb666837e3txhhjgsGgWb16dbRfRUWFmTt3rpk7d6759a9/3WecpqYmfUv0DBpMXZqbm83EiRNNTk6OycvLM3l5eeaZZ545J+sYDl588UUzb948k52dbR555BFjjDEbNmwwe/fuNcYY09nZaW655RYzd+5cU1BQYBobG6OffeSRR0x2draZN2+evql7Bn3emjz88MNm6tSp0X8XeXl55v333z9n6xhuBvNv5RObNm3St0QHyGHM376yISIiIiK2pCNREREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZtTYBMRERGxOQU2EREREZv7f2zhJgI6YT6qAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 4))\n",
    "sns.lineplot(range(len(history['train_loss'])), history['train_loss'])\n",
    "plt.title('Train loss')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "sns.lineplot(range(len(history['val_accuracy'])), history['val_accuracy'])\n",
    "plt.title('Validation loss')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "sns.lineplot(range(len(history['train_loss_stds'])), history['train_loss_stds'])\n",
    "plt.title('Train loss std')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "sns.lineplot(range(len(history['val_accuracy_stds'])), history['val_accuracy_stds'])\n",
    "plt.title('Validation loss std')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SUBMIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = nflrush.make_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def get_predictions(df_test, y_test, model):\n",
    "    X = make_x(preprocess_features(df_test, fillna=True))\n",
    "    y_pred = nn_pipeline_predict(model, scaler, X)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess_features(df_test).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(make_x(df_train_preprocessed).columns).difference(set(make_x(preprocess_features(df_test, fillna=-999))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# set(make_x(preprocess_features(df_test, fillna=True)).columns).difference(set(make_x(df_train_preprocessed).columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "iter_test = env.iter_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "380it [00:47,  8.41it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-9d7deecf7d62>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_cdf_test\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_predictions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_cdf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_cdf_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-40-b4caa37f283b>\u001b[0m in \u001b[0;36mget_predictions\u001b[0;34m(df_test, y_test, model)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_predictions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_x\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreprocess_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfillna\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn_pipeline_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscaler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-31-7c2cdd9e989b>\u001b[0m in \u001b[0;36mpreprocess_features\u001b[0;34m(df, fillna)\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeHandoff'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeHandoff'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeSnap'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeSnap'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeDelta'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeHandoff'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TimeSnap'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_seconds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'PlayerBirthDate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'PlayerBirthDate'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrptime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"%m/%d/%Y\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'PlayerBirthDate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_datetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'PlayerBirthDate'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   3589\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3590\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3591\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3592\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3593\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for df_test, y_cdf_test in tqdm.tqdm(iter_test):\n",
    "    y_pred = get_predictions(df_test, y_cdf_test, model)\n",
    "    env.predict(pd.DataFrame(data=y_pred,columns=y_cdf_test.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.write_submission_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}