Titanic_NB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pre_processing\n",
    "from sklearn.base import BaseEstimator,TransformerMixin\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import make_column_transformer, ColumnTransformer\n",
    "\n",
    "# selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "# models\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "import xgboost as xgb\n",
    "from catboost import CatBoostClassifier, Pool, cv\n",
    "import lightgbm as lgb\n",
    "\n",
    "# metrics\n",
    "from sklearn.metrics import f1_score, plot_roc_curve, auc\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import scipy\n",
    "from sklearn.utils.fixes import loguniform\n",
    "from scipy.stats import uniform\n",
    "\n",
    "# general\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from time import time\n",
    "from Titanic_utils import *\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams.update({'text.color' : \"k\",\n",
    "                     'axes.labelcolor' : \"w\",\n",
    "                     'xtick.color' : \"w\",\n",
    "                     'ytick.color' : \"w\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we need to be able to reproduce features from the train to the test,\n",
    "so we'll create a custom sklean preprocessor transformer object.\n",
    "[inspiration](https://scikit-learn.org/stable/auto_examples/compose/plot_column_transformer_mixed_types.html#sphx-glr-auto-examples-compose-plot-column-transformer-mixed-types-py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Preprocessor(BaseEstimator,TransformerMixin):\n",
    "    \"\"\"\n",
    "    A preprocessor transformer class for the Titanic dataset.\n",
    "    The pre-processing stages are as follows:\n",
    "    - Use title of names as a feature.\n",
    "    - Fill NaN ages by the mean age grouped by Title,\n",
    "        the highest correlating feature.\n",
    "    - parse deck level from 'Cabin' feature.\n",
    "    - drop Name, Cabin, Ticket as they were used for another feature,\n",
    "        or aren't worth the trouble.\n",
    "    - one-hot categorical data.\n",
    "    - possibly add/remove columns, based on feature exploration\n",
    "        and false analysis (added ability to define what columns to drop,\n",
    "        if they prove to be bad).\n",
    "    \"\"\"\n",
    "\n",
    "    def fit(self, X, *y):\n",
    "        \"\"\"\n",
    "        fit the preprocessor parameters to the training data.\n",
    "        :param X: data\n",
    "        :param y: labels\n",
    "        :return: none (transformer is now fitted).\n",
    "        \"\"\"\n",
    "        X = X.copy()\n",
    "        X.reset_index(inplace=True,drop=True)\n",
    "        # create Rare title for titles without enough examples (we consider\n",
    "        # it to be unknown).\n",
    "        X['Title'] = X['Name'].apply(lambda x: x.split(',')[1].split()[0])\n",
    "        self.freq_titles = X['Title'].value_counts()\n",
    "        self.freq_titles = self.freq_titles[self.freq_titles>4].index\n",
    "        X.loc[~X['Title'].isin(self.freq_titles),'Title'] = 'Rare'\n",
    "        self.ages = X.groupby('Title')['Age'].mean()\n",
    "        splitted_cabin = X['Cabin'].dropna().str.split(pat='[\\d ]').apply(lambda x: x[0])\n",
    "        X['Cabin_Deck'] = pd.DataFrame(splitted_cabin.tolist(), index = splitted_cabin.index)\n",
    "        if(self.one_hot_cat):\n",
    "            self.ohe = OneHotEncoder(handle_unknown='ignore',sparse=False)\n",
    "            self.ohe.fit(X[self.cat_columns].fillna(-1).astype(str))\n",
    "        else:\n",
    "            # catboost/LightGBM\n",
    "            X[self.cat_text_cols] = X[self.cat_text_cols].fillna('nan')\n",
    "            self.ord_enc = OrdinalEncoderWithUnknown()\n",
    "            self.ord_enc.fit(X[self.cat_text_cols])\n",
    "        # addition\n",
    "        X['SibSp^2'] = X['SibSp']**2\n",
    "        self.scaler = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('num', StandardScaler(), ['SibSp^2','Fare','Age'])],remainder='drop')\n",
    "        self.scaler.fit(X[['SibSp^2','Fare','Age']])\n",
    "\n",
    "    def add_columns_to_drop(self,cols):\n",
    "        self.drop_columns.append(cols)\n",
    "\n",
    "    def remove_columns_to_drop(self,cols):\n",
    "        self.drop_columns.remove(cols)\n",
    "\n",
    "    def drop_try(self,df):\n",
    "        '''\n",
    "        try to drop columns in 'self.columns_to_drop'.\n",
    "        if columns doesn't exist - do nothing.\n",
    "        :param df:\n",
    "        :return:\n",
    "        '''\n",
    "        for col in self.drop_columns:\n",
    "         try:\n",
    "             df.drop(col,axis=1,inplace=True)\n",
    "         except:\n",
    "             pass\n",
    "        return df\n",
    "\n",
    "    def transform(self,X, *y):\n",
    "        \"\"\"\n",
    "        transform the test/cv data to the same format as the train data.\n",
    "        :param X: data\n",
    "        :param y: none (unused for preprocessing).\n",
    "        :return: preprocessed data.\n",
    "        \"\"\"\n",
    "        X = X.copy()\n",
    "        X.reset_index(inplace=True,drop=True)\n",
    "        # ticket: just shows family relations, which is covered in parch and sibsp anyway.\n",
    "        # name: we already extracted titles\n",
    "        # cabin: we used it for deck feature, no need for it afterwards.\n",
    "        # one_hot categories all at once, drop originals afterwards.\n",
    "        # name_to_titles\n",
    "        X['Title'] = X['Name'].apply(lambda x: x.split(',')[1].split()[0])\n",
    "        X.loc[~X['Title'].isin(self.freq_titles),'Title'] = 'Rare'\n",
    "        # fill age NaNs by titles\n",
    "        # X.Age = self.age_imputer.transform(X.select_dtypes(exclude=object))[:,1]\n",
    "        X.Age = X.apply(lambda x : self.ages[x['Title']] if np.isnan(x['Age']) else x['Age'],axis=1)\n",
    "        # cabin_to_deck_no (keep NaNs as is, column is cabin_deck_-1)\n",
    "        splitted_cabin = X['Cabin'].dropna().str.split(pat='[\\d ]').apply(lambda x: x[0])\n",
    "        X['Cabin_Deck'] = pd.DataFrame(splitted_cabin.tolist(), index = splitted_cabin.index)\n",
    "        # as sex has only 2 categories, just binarize it.\n",
    "        X['Sex'] = (X['Sex']=='male').astype(int)\n",
    "        # found out from feature exploration that sibsp^2 makes a difference.\n",
    "        X['SibSp^2'] = X['SibSp']**2\n",
    "        # one_hot categories all at once, drop originals afterwards.\n",
    "        if(self.one_hot_cat):\n",
    "            ohe_array = self.ohe.transform(X[self.cat_columns].fillna(-1).astype(str))\n",
    "            ohe_df = pd.DataFrame(ohe_array, columns=self.ohe.get_feature_names(self.cat_columns))\n",
    "            X = pd.concat([X,ohe_df],axis='columns').drop(self.cat_columns,axis='columns')\n",
    "        else:\n",
    "            X[self.cat_text_cols] = self.ord_enc.transform(X[self.cat_text_cols].fillna('nan')).astype(int)\n",
    "        X[['SibSp^2','Fare','Age']] = self.scaler.transform(X[['SibSp^2','Fare','Age']])\n",
    "        X = self.drop_try(X)\n",
    "        self.features = X.columns\n",
    "        return X\n",
    "\n",
    "    def fit_transform(self,X, *y):\n",
    "        \"\"\"\n",
    "        fit the preprocessor parameters to the training data,\n",
    "        and transform the train data as well (using tranform func)\n",
    "        :param data:\n",
    "        X_train data, to fit the data preprocessor transformer.\n",
    "        :return:\n",
    "        -------\n",
    "        X_new : ndarray array of shape (n_samples, n_features_new)\n",
    "            Transformed array.\n",
    "        \"\"\"\n",
    "        self.fit(X,y)\n",
    "        return self.transform(X,y)\n",
    "    def get_features(self):\n",
    "        return self.features\n",
    "    def __init__(self,one_hot_cat = True,cols_to_drop = None):\n",
    "        self.one_hot_cat = one_hot_cat\n",
    "        self.drop_columns = ['Ticket','Name','Cabin']\n",
    "        if cols_to_drop is not None :\n",
    "            self.drop_columns += cols_to_drop\n",
    "        self.cat_columns = ['Pclass','Title','SibSp','Embarked','Cabin_Deck']\n",
    "        self.cat_text_cols = ['Title','Embarked','Cabin_Deck']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data and classifier setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('titanic.csv')\n",
    "data.dtypes\n",
    "X, X_test, y, y_test = train_test_split(data.drop('Survived',axis=1),\n",
    "                                        data['Survived'], test_size=0.2,\n",
    "                                        random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Value\n",
    "method: just through away everything that isn't numeric,\n",
    "fill with median (robust to outliers), One_hot categories, done.\n",
    "taken from [here](https://scikit-learn.org/stable/auto_examples/compose/plot_column_transformer_mixed_types.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline scores (cv and train): \n",
      "[0.80295464 0.77913025 0.77293771 0.71675084 0.79128683]\n",
      "0.7726120561847719\n",
      "[0.78336618 0.77082512 0.77566328 0.7901736  0.77581607]\n",
      "0.7791688508252099\n"
     ]
    }
   ],
   "source": [
    "numeric_features = ['Age', 'Fare']\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='median')),\n",
    "    ('scaler', RobustScaler())])\n",
    "categorical_features = ['Pclass','Sex','SibSp','Parch','Embarked']\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n",
    "    ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "# note that Col Transformer returns only the transformed columns (unless remainder param set to 'passthrough')\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', numeric_transformer, numeric_features),\n",
    "        ('cat', categorical_transformer, categorical_features)])\n",
    "baseline_clf = Pipeline(steps=[('preprocessor', preprocessor),\n",
    "                      ('classifier', LogisticRegression())])\n",
    "\n",
    "scores = cross_val_score_regular(baseline_clf,X,y)\n",
    "print('baseline scores (cv and train): ',*scores,sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sex                int32\n",
       "Age              float64\n",
       "Parch              int64\n",
       "Fare             float64\n",
       "SibSp^2          float64\n",
       "Pclass_1         float64\n",
       "Pclass_2         float64\n",
       "Pclass_3         float64\n",
       "Title_Dr.        float64\n",
       "Title_Master.    float64\n",
       "Title_Miss.      float64\n",
       "Title_Mr.        float64\n",
       "Title_Mrs.       float64\n",
       "Title_Rare       float64\n",
       "SibSp_0          float64\n",
       "SibSp_1          float64\n",
       "SibSp_2          float64\n",
       "SibSp_3          float64\n",
       "SibSp_4          float64\n",
       "SibSp_5          float64\n",
       "SibSp_8          float64\n",
       "Embarked_-1      float64\n",
       "Embarked_C       float64\n",
       "Embarked_Q       float64\n",
       "Embarked_S       float64\n",
       "Cabin_Deck_-1    float64\n",
       "Cabin_Deck_A     float64\n",
       "Cabin_Deck_B     float64\n",
       "Cabin_Deck_C     float64\n",
       "Cabin_Deck_D     float64\n",
       "Cabin_Deck_E     float64\n",
       "Cabin_Deck_F     float64\n",
       "Cabin_Deck_G     float64\n",
       "Cabin_Deck_T     float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp = Preprocessor()\n",
    "pp.fit(X)\n",
    "pp.transform(X).dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# general structure of pipeline:\n",
    "from sklearn import set_config\n",
    "set_config(display='diagram')\n",
    "baseline_clf\n",
    "set_config(display='text')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## catboost and lightgbm\n",
    " they can handle categorical data, so we'll start off with them, as they need a special setup."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "catboost cv and train scores: \n",
      "[0.77517686 0.8141906  0.78430135 0.77861953 0.83644265]\n",
      "0.7977461973990487\n",
      "[0.88446328 0.86012351 0.87232231 0.86858827 0.85734463]\n",
      "0.8685683989629538\n",
      "lgbm cv and train scores: \n",
      "[0.77517686 0.8363504  0.80639731 0.78787879 0.78927284]\n",
      "0.7990152389558791\n",
      "[0.96129287 0.94410064 0.94546348 0.95018015 0.94985876]\n",
      "0.9501791791347216\n"
     ]
    }
   ],
   "source": [
    "cb_clf = CatBoostClassifier(iterations=100,\n",
    "                            task_type=\"GPU\",\n",
    "                            devices='0=1',\n",
    "                            cat_features=['Pclass','Title','SibSp',\n",
    "                                          'Parch','Embarked','Cabin_Deck'],\n",
    "                            verbose=False)\n",
    "# cb = False\n",
    "# if(cb):\n",
    "pipe = Pipeline([('pre_processing',Preprocessor(one_hot_cat=False)),('classifier',cb_clf)])\n",
    "cv_scores = cross_val_score_regular(pipe,X,y)\n",
    "print('catboost cv and train scores: ',*cv_scores,sep='\\n')\n",
    "# else:\n",
    "lgb_params = {'classifier__categorical_feature':['Pclass','Title','SibSp','Parch','Embarked','Cabin_Deck']}\n",
    "lgb_clf = lgb.sklearn.LGBMClassifier()\n",
    "pipe = Pipeline([('pre_processing',Preprocessor(one_hot_cat=False)),('classifier',lgb_clf)])\n",
    "pipe.fit(X,y)\n",
    "cv_scores = cross_val_score_regular(pipe,X,y)\n",
    "print('lgbm cv and train scores: ',*cv_scores,sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fitting classifiers with hyper-parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "setting up classifiers and their parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "names = [\"Logistic_Regression\",\n",
    "         \"Linear_SVM\", \"RBF_SVM\",\n",
    "         \"Decision_Tree\",\n",
    "         \"Neural_Net\", \"AdaBoost\",\n",
    "         \"Random_Forest\",\"XGBoost\"]\n",
    "param_grid = {'Logistic_Regression':\n",
    "                  {'classifier__C':loguniform(1e-3, 1e3),\n",
    "                   'classifier__penalty':['l1','l2']},\n",
    "              'Linear_SVM':\n",
    "                  {'classifier__C': scipy.stats.expon(scale=100),\n",
    "                    'classifier__gamma': scipy.stats.expon(scale=.1),\n",
    "                    'classifier__kernel': ['rbf'],\n",
    "                    'classifier__class_weight':['balanced', None]},\n",
    "              'RBF_SVM':\n",
    "                  {'classifier__C': loguniform(1e0, 1e3),\n",
    "                    'classifier__gamma': loguniform(1e-4, 1e-3),\n",
    "                    'classifier__kernel': ['rbf']},\n",
    "              'Decision_Tree':\n",
    "                  {\"classifier__max_depth\": np.linspace(2,10,5),\n",
    "                     \"classifier__max_features\": range(1,9),\n",
    "                     \"classifier__min_samples_leaf\": range(3,9),\n",
    "                     \"classifier__criterion\": [\"gini\", \"entropy\"]},\n",
    "              'Neural_Net':{\n",
    "                  'classifier__solver': ['lbfgs','adam'],\n",
    "                  'classifier__max_iter': [500,1000,1500],\n",
    "                  'classifier__alpha': 10.0 ** -np.arange(1, 7),\n",
    "                  'classifier__hidden_layer_sizes':np.arange(5, 12)\n",
    "              },\n",
    "              'AdaBoost':{\n",
    "                  'classifier__n_estimators': [100,200,500],\n",
    "                  'classifier__learning_rate' : loguniform(1e-2, 1e0),\n",
    "                  'classifier__base_estimator__max_depth' : [4,5,6,7,8]\n",
    "              },\n",
    "              'Random_Forest':{\n",
    "                  'classifier__n_estimators': [200, 500],\n",
    "                  'classifier__max_features': ['auto', 'sqrt', 'log2'],\n",
    "                  'classifier__max_depth' : [4,5,6,7,8],\n",
    "                  'classifier__criterion' :['gini', 'entropy']},\n",
    "              'XGBoost':{\n",
    "                  'classifier__learning_rate':loguniform(1e-2, 1e0),\n",
    "                  'classifier__gamma':[0,1,5],\n",
    "                  'classifier__n_estimators': [500,750,1000],\n",
    "                  'classifier__max_depth': [3,6,8],\n",
    "                  'classifier__subsample': uniform(loc=0.8,scale=0.2),\n",
    "                  'classifier__colsample_bytree': uniform(loc=0.8,scale=0.2)\n",
    "              }}\n",
    "classifiers = [\n",
    "    LogisticRegression(C=0.3,n_jobs=-1,solver='newton-cg'),\n",
    "    SVC(kernel=\"linear\", C=0.025),\n",
    "    SVC(gamma=2, C=1),\n",
    "    DecisionTreeClassifier(max_depth=5),\n",
    "    MLPClassifier(hidden_layer_sizes=(5,3),alpha=1, max_iter=1000),\n",
    "    AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=10),n_estimators=500),\n",
    "    RandomForestClassifier(max_depth=9, n_estimators=400, max_features=1),\n",
    "    xgb.XGBClassifier(max_depth=10,learning_rate =0.3,n_estimators=500,\n",
    "                      objective=\"reg:squarederror\", n_jobs = -1,random_state=42),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Random Search on each Clf (20 rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic_Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 6.72 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 LogisticRegression(C=0.10619108399169305, n_jobs=-1,\n",
      "                                    solver='newton-cg'))]) \n",
      "auc:  0.8625159222427083\n",
      "Linear_SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 4.21 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 SVC(C=16.493994305284705, gamma=0.0190010899985659))]) \n",
      "auc:  0.8624328267838444\n",
      "RBF_SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 3.86 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 SVC(C=25.306063066623494, gamma=0.00012337949527837963))]) \n",
      "auc:  0.8566019667739511\n",
      "Decision_Tree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 3.34 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 DecisionTreeClassifier(max_depth=8.0, max_features=8,\n",
      "                                        min_samples_leaf=6))]) \n",
      "auc:  0.8290599175987134\n",
      "Neural_Net\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 19.76 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 MLPClassifier(alpha=1e-06, hidden_layer_sizes=8,\n",
      "                               max_iter=1500))]) \n",
      "auc:  0.8613236233082004\n",
      "AdaBoost\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 19.30 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=4),\n",
      "                                    learning_rate=0.021080020568984736,\n",
      "                                    n_estimators=500))]) \n",
      "auc:  0.838199994253895\n",
      "Random_Forest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 16.74 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 RandomForestClassifier(max_depth=6, max_features='log2',\n",
      "                                        n_estimators=200))]) \n",
      "auc:  0.86191175588833\n",
      "XGBoost\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n",
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV took 28.20 seconds for 20 candidates parameter settings.\n",
      "Pipeline(steps=[('pre_processing', Preprocessor()),\n",
      "                ('classifier',\n",
      "                 XGBClassifier(base_score=0.5, booster='gbtree',\n",
      "                               colsample_bylevel=1, colsample_bynode=1,\n",
      "                               colsample_bytree=0.8162398049146752, gamma=1,\n",
      "                               gpu_id=-1, importance_type='gain',\n",
      "                               interaction_constraints='',\n",
      "                               learning_rate=0.0592178042115199,\n",
      "                               max_delta_step=0, max_depth=3,\n",
      "                               min_child_weight=1, missing=nan,\n",
      "                               monotone_constraints='()', n_estimators=500,\n",
      "                               n_jobs=-1, num_parallel_tree=1,\n",
      "                               objective='reg:squarederror', random_state=42,\n",
      "                               reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
      "                               subsample=0.8025175231674534,\n",
      "                               tree_method='exact', validate_parameters=1,\n",
      "                               verbosity=None))]) \n",
      "auc:  0.8634841587738027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ariel\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\sklearn\\base.py:209: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
      "  warnings.warn('From version 0.24, get_params will raise an '\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "random_searches = {}\n",
    "for (name,clf) in zip(names, classifiers):\n",
    "    pipeclf = Pipeline([('pre_processing',Preprocessor()),('classifier',clf)])\n",
    "    print(name)\n",
    "    n_iter_search = 20\n",
    "    random_searches[name] = RandomizedSearchCV(pipeclf,param_grid[name],\n",
    "                                               n_jobs=-1,scoring='roc_auc',\n",
    "                                               n_iter=n_iter_search,\n",
    "                                               return_train_score=True)\n",
    "    start = time()\n",
    "    random_search = random_searches[name]\n",
    "    random_search.fit(X, y)\n",
    "    print(\"RandomizedSearchCV took %.2f seconds for %d candidates\"\n",
    "          \" parameter settings.\" % ((time() - start), n_iter_search))\n",
    "    print(random_search.best_estimator_,'\\nauc: ',random_search.best_score_)\n",
    "    scores.append((name,random_search.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# False analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['random_searches.joblib']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save searches.\n",
    "from joblib import dump, load\n",
    "dump(random_searches, 'random_searches.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "random_searches = load('random_searches.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "see scores of the best classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost \n",
      "\n",
      "best index:  2\n",
      "mean_fit_time 0.6515990734100342\n",
      "std_fit_time 0.012449873920408389\n",
      "mean_score_time 0.04359965324401856\n",
      "std_score_time 0.0050041577898722895\n",
      "param_classifier__colsample_bytree 0.8162398049146752\n",
      "param_classifier__gamma 1\n",
      "param_classifier__learning_rate 0.0592178042115199\n",
      "param_classifier__max_depth 3\n",
      "param_classifier__n_estimators 500\n",
      "param_classifier__subsample 0.8025175231674534\n",
      "params {'classifier__colsample_bytree': 0.8162398049146752, 'classifier__gamma': 1, 'classifier__learning_rate': 0.0592178042115199, 'classifier__max_depth': 3, 'classifier__n_estimators': 500, 'classifier__subsample': 0.8025175231674534}\n",
      "split0_test_score 0.8315647107781939\n",
      "split1_test_score 0.9204119850187266\n",
      "split2_test_score 0.8207070707070707\n",
      "split3_test_score 0.8450126262626263\n",
      "split4_test_score 0.8997244011023956\n",
      "mean_test_score 0.8634841587738027\n",
      "std_test_score 0.039354950138787224\n",
      "rank_test_score 1\n",
      "split0_train_score 0.9204046774405465\n",
      "split1_train_score 0.890224674812771\n",
      "split2_train_score 0.9077169996724535\n",
      "split3_train_score 0.9110907304290861\n",
      "split4_train_score 0.8978081188533166\n",
      "mean_train_score 0.9054490402416349\n",
      "std_train_score 0.010495793200616499\n",
      "\n",
      "\n",
      "\n",
      "Logistic_Regression \n",
      "\n",
      "best index:  13\n",
      "mean_fit_time 0.15779848098754884\n",
      "std_fit_time 0.02521621114898199\n",
      "mean_score_time 0.060804891586303714\n",
      "std_score_time 0.00932395535863312\n",
      "param_classifier__C 0.10619108399169305\n",
      "param_classifier__penalty l2\n",
      "params {'classifier__C': 0.10619108399169305, 'classifier__penalty': 'l2'}\n",
      "split0_test_score 0.8295880149812733\n",
      "split1_test_score 0.8884727424053267\n",
      "split2_test_score 0.8364898989898989\n",
      "split3_test_score 0.8536405723905724\n",
      "split4_test_score 0.9043883824464702\n",
      "mean_test_score 0.8625159222427083\n",
      "std_test_score 0.029214490358181456\n",
      "rank_test_score 1\n",
      "split0_train_score 0.8804362107476023\n",
      "split1_train_score 0.8652082512153463\n",
      "split2_train_score 0.8791811333114969\n",
      "split3_train_score 0.8735669832951196\n",
      "split4_train_score 0.8635043942247331\n",
      "mean_train_score 0.8723793945588596\n",
      "std_train_score 0.006968173820288446\n",
      "\n",
      "\n",
      "\n",
      "Linear_SVM \n",
      "\n",
      "best index:  3\n",
      "mean_fit_time 0.13879947662353515\n",
      "std_fit_time 0.012796167276123227\n",
      "mean_score_time 0.050000905990600586\n",
      "std_score_time 0.002899118579077344\n",
      "param_classifier__C 16.493994305284705\n",
      "param_classifier__class_weight None\n",
      "param_classifier__gamma 0.0190010899985659\n",
      "param_classifier__kernel rbf\n",
      "params {'classifier__C': 16.493994305284705, 'classifier__class_weight': None, 'classifier__gamma': 0.0190010899985659, 'classifier__kernel': 'rbf'}\n",
      "split0_test_score 0.818976279650437\n",
      "split1_test_score 0.8924261339991677\n",
      "split2_test_score 0.8541666666666666\n",
      "split3_test_score 0.851746632996633\n",
      "split4_test_score 0.8948484206063175\n",
      "mean_test_score 0.8624328267838444\n",
      "std_test_score 0.028359706733586093\n",
      "rank_test_score 1\n",
      "split0_train_score 0.8959138089607147\n",
      "split1_train_score 0.8886480094599922\n",
      "split2_train_score 0.8930167048804455\n",
      "split3_train_score 0.8980674746151327\n",
      "split4_train_score 0.893243879472693\n",
      "mean_train_score 0.8937779754777957\n",
      "std_train_score 0.0031667942883884265\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "random_searches_list = list(random_searches.items())\n",
    "random_searches_list.sort(key=lambda x:x[1].best_score_,reverse=True)\n",
    "for (name,rand_search) in random_searches_list[:3]:\n",
    "    print(name,'\\n')\n",
    "    print('best index: ',rand_search.best_index_)\n",
    "    for metric in rand_search.cv_results_.items():\n",
    "        print(metric[0],metric[1][rand_search.best_index_])\n",
    "    print('\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Columns Importances\n",
      "0             Sex     0.17875\n",
      "1             Age   0.0321568\n",
      "2           Parch   0.0218493\n",
      "3            Fare   0.0437958\n",
      "4         SibSp^2   0.0443479\n",
      "5        Pclass_1   0.0164144\n",
      "6        Pclass_2           0\n",
      "7        Pclass_3    0.117786\n",
      "8       Title_Dr.   0.0299772\n",
      "9   Title_Master.    0.044601\n",
      "10    Title_Miss.           0\n",
      "11      Title_Mr.    0.284978\n",
      "12     Title_Mrs.   0.0244918\n",
      "13     Title_Rare           0\n",
      "14        SibSp_0           0\n",
      "15        SibSp_1           0\n",
      "16        SibSp_2           0\n",
      "17        SibSp_3           0\n",
      "18        SibSp_4           0\n",
      "19        SibSp_5           0\n",
      "20        SibSp_8           0\n",
      "21    Embarked_-1           0\n",
      "22     Embarked_C    0.030049\n",
      "23     Embarked_Q           0\n",
      "24     Embarked_S    0.037153\n",
      "25  Cabin_Deck_-1   0.0574137\n",
      "26   Cabin_Deck_A           0\n",
      "27   Cabin_Deck_B           0\n",
      "28   Cabin_Deck_C           0\n",
      "29   Cabin_Deck_D           0\n",
      "30   Cabin_Deck_E    0.036236\n",
      "31   Cabin_Deck_F           0\n",
      "32   Cabin_Deck_G           0\n",
      "33   Cabin_Deck_T           0\n"
     ]
    }
   ],
   "source": [
    "best_clf = random_searches_list[0][1].best_estimator_\n",
    "cols = best_clf['pre_processing'].fit_transform(X).columns\n",
    "importances = best_clf['classifier'].feature_importances_\n",
    "df = pd.DataFrame(np.c_[cols,importances],columns=['Columns','Importances'])\n",
    "print(df)\n",
    "# print(list(df[df['Importances']==0]['Columns']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[81  8]\n",
      " [16 38]]\n",
      "fp: \n",
      "      Pclass                                               Name     Sex   Age  \\\n",
      "7         2              Lahtinen, Mrs. William (Anna Sylfven)  female  26.0   \n",
      "35        1                          Brewe, Dr. Arthur Jackson    male   NaN   \n",
      "36        3                     Palsson, Master. Gosta Leonard    male   2.0   \n",
      "42        2             Yrois, Miss. Henriette (\"Mrs Harbeck\")  female  24.0   \n",
      "52        3                                Canavan, Miss. Mary  female  21.0   \n",
      "114       3                      Boulos, Mrs. Joseph (Sultana)  female   NaN   \n",
      "128       3     Robins, Mrs. Alexander A (Grace Charity Laury)  female  47.0   \n",
      "132       2  Turpin, Mrs. William John Robert (Dorothy Ann ...  female  27.0   \n",
      "\n",
      "     SibSp  Parch     Ticket     Fare Cabin Embarked  \n",
      "7        1      1     250651  26.0000   NaN        S  \n",
      "35       0      0     112379  39.6000   NaN        C  \n",
      "36       3      1     349909  21.0750   NaN        S  \n",
      "42       0      0     248747  13.0000   NaN        S  \n",
      "52       0      0     364846   7.7500   NaN        Q  \n",
      "114      0      2       2678  15.2458   NaN        C  \n",
      "128      1      0  A/5. 3337  14.5000   NaN        S  \n",
      "132      1      0      11668  21.0000   NaN        S  \n",
      "\n",
      "\n",
      "fn: \n",
      "      Pclass                                Name     Sex   Age  SibSp  Parch  \\\n",
      "6         3             de Mulder, Mr. Theodore    male  30.0      0      0   \n",
      "9         1                   Woolner, Mr. Hugh    male   NaN      0      0   \n",
      "14        3         Sheerlinck, Mr. Jan Baptist    male  29.0      0      0   \n",
      "27        1            Taylor, Mr. Elmer Zebley    male  48.0      1      0   \n",
      "31        1    Frolicher-Stehli, Mr. Maxmillian    male  60.0      1      1   \n",
      "57        3        McCormack, Mr. Thomas Joseph    male   NaN      0      0   \n",
      "60        1     Homer, Mr. Harry (\"Mr E Haven\")    male  35.0      0      0   \n",
      "64        3                     Foo, Mr. Choong    male   NaN      0      0   \n",
      "71        3            Dowdell, Miss. Elizabeth  female  30.0      0      0   \n",
      "82        2                   Beane, Mr. Edward    male  32.0      1      0   \n",
      "84        1            Goldenberg, Mr. Samuel L    male  49.0      1      0   \n",
      "88        3  Pickard, Mr. Berk (Berk Trembisky)    male  32.0      0      0   \n",
      "112       1                Marechal, Mr. Pierre    male   NaN      0      0   \n",
      "118       3   Johannesen-Bratthammer, Mr. Bernt    male   NaN      0      0   \n",
      "125       3      Sunderland, Mr. Victor Francis    male  16.0      0      0   \n",
      "142       3               Dahl, Mr. Karl Edwart    male  45.0      0      0   \n",
      "\n",
      "                Ticket     Fare Cabin Embarked  \n",
      "6               345774   9.5000   NaN        S  \n",
      "9                19947  35.5000   C52        S  \n",
      "14              345779   9.5000   NaN        S  \n",
      "27               19996  52.0000  C126        S  \n",
      "31               13567  79.2000   B41        C  \n",
      "57              367228   7.7500   NaN        Q  \n",
      "60              111426  26.5500   NaN        C  \n",
      "64                1601  56.4958   NaN        S  \n",
      "71              364516  12.4750   NaN        S  \n",
      "82                2908  26.0000   NaN        S  \n",
      "84               17453  89.1042   C92        C  \n",
      "88   SOTON/O.Q. 392078   8.0500   E10        S  \n",
      "112              11774  29.7000   C47        C  \n",
      "118              65306   8.1125   NaN        S  \n",
      "125    SOTON/OQ 392089   8.0500   NaN        S  \n",
      "142               7598   8.0500   NaN        S  \n",
      "train score:  0.8558862173170412\n",
      "cv score:  0.8069080316271328 \n",
      "\n",
      "[[82  7]\n",
      " [14 40]]\n",
      "train score:  0.8205886217317041\n",
      "cv score:  0.8310445276737412 \n",
      "\n",
      "[[77 11]\n",
      " [16 38]]\n",
      "train score:  0.8479528332787423\n",
      "cv score:  0.7893518518518519 \n",
      "\n",
      "[[83  5]\n",
      " [20 34]]\n",
      "train score:  0.8430068784801833\n",
      "cv score:  0.7864057239057239 \n",
      "\n",
      "[[77 12]\n",
      " [12 41]]\n",
      "train score:  0.8363543628374137\n",
      "cv score:  0.8193767224931101 \n",
      "\n",
      "\n",
      "\n",
      "train scores:  [0.85588622 0.82058862 0.84795283 0.84300688 0.83635436] \n",
      "mean:  0.8407577827290169\n",
      "cv scores:  [0.80690803 0.83104453 0.78935185 0.78640572 0.81937672] \n",
      "mean:  0.8066173715103119\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.12554821372032166, 'Sex'],\n",
       "       [0.027515694499015808, 'Age'],\n",
       "       [0.018225906416773796, 'Parch'],\n",
       "       [0.03561985492706299, 'Fare'],\n",
       "       [0.03603928163647652, 'SibSp^2'],\n",
       "       [0.05899054557085037, 'Pclass_1'],\n",
       "       [0.0656723603606224, 'Pclass_2'],\n",
       "       [0.07947859913110733, 'Pclass_3'],\n",
       "       [0.0, 'Title_Dr.'],\n",
       "       [0.0446961335837841, 'Title_Master.'],\n",
       "       [0.008875944651663303, 'Title_Miss.'],\n",
       "       [0.224235400557518, 'Title_Mr.'],\n",
       "       [0.03947775810956955, 'Title_Mrs.'],\n",
       "       [0.03019719384610653, 'Title_Rare'],\n",
       "       [0.0, 'SibSp_0'],\n",
       "       [0.0, 'SibSp_1'],\n",
       "       [0.0, 'SibSp_2'],\n",
       "       [0.0, 'SibSp_3'],\n",
       "       [0.0, 'SibSp_4'],\n",
       "       [0.0, 'SibSp_5'],\n",
       "       [0.0, 'SibSp_8'],\n",
       "       [0.0, 'Embarked_-1'],\n",
       "       [0.021867332980036736, 'Embarked_C'],\n",
       "       [0.0, 'Embarked_Q'],\n",
       "       [0.031197141855955124, 'Embarked_S'],\n",
       "       [0.024834902957081795, 'Cabin_Deck_-1'],\n",
       "       [0.03325647488236427, 'Cabin_Deck_A'],\n",
       "       [0.0, 'Cabin_Deck_B'],\n",
       "       [0.01499619148671627, 'Cabin_Deck_C'],\n",
       "       [0.05008317902684212, 'Cabin_Deck_D'],\n",
       "       [0.029191957786679268, 'Cabin_Deck_E'],\n",
       "       [0.0, 'Cabin_Deck_F'],\n",
       "       [0.0, 'Cabin_Deck_G'],\n",
       "       [0.0, 'Cabin_Deck_T']], dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importances,fp,fn = get_errors_and_features(best_clf,X,y)\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "We see that most misclassifications are fn.\n",
    "let's take a closer look:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     1.0\n",
       "1     1.0\n",
       "2     1.0\n",
       "3     1.0\n",
       "4     1.0\n",
       "5     1.0\n",
       "6     1.0\n",
       "7     1.0\n",
       "8     0.0\n",
       "9     1.0\n",
       "10    1.0\n",
       "11    1.0\n",
       "12    1.0\n",
       "13    1.0\n",
       "14    1.0\n",
       "15    1.0\n",
       "Name: Title_Mr., dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_clf['pre_processing'].transform(fn)['Title_Mr.']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "we see that almost all fn's are men ('Mr.' to be specific). In addition,\n",
    "we see that the classifier puts a large emphasis on both title and one's Sex.\\\n",
    "Since we can assume the Titles cover one's sex (a 'Mrs.' is a female, etc),\\\n",
    "we'll try to remove the Sex, as it has a high correlation with the titles.\n",
    "to prove this, we'll check it's chi2 score with \"Mr.\":"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([218.97435897]), array([1.51392505e-49]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_x = best_clf['pre_processing'].transform(X)\n",
    "chi2(trans_x['Title_Mr.'].to_frame(),trans_x['Sex'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "voalá. Insane score. let's try the same thing without Sex (as the titles\n",
    "are more precise than Sex):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original score, with sex:\n",
      "\n",
      "[0.80129005 0.83104453 0.78935185 0.77714646 0.81937672]\n",
      "0.8036419241528421\n",
      "[0.85447379 0.81917619 0.84189322 0.84624959 0.83635436]\n",
      "0.8396294306959143\n",
      "\n",
      "\n",
      "\"improved\" score, without sex:\n",
      "\n",
      "[0.80129005 0.82542655 0.78009259 0.77861953 0.81757473]\n",
      "0.8006006910315794\n",
      "[0.85630009 0.82183025 0.8414019  0.83449066 0.84188638]\n",
      "0.8391818560525918\n"
     ]
    }
   ],
   "source": [
    "trans_x_no_sex = trans_x.drop('Sex',axis=1)\n",
    "print('original score, with sex:\\n',\n",
    "      *cross_val_score_regular(best_clf['classifier'],trans_x,y),sep='\\n')\n",
    "print('\\n\\n\"improved\" score, without sex:\\n',\n",
    "      *cross_val_score_regular(best_clf['classifier'],trans_x_no_sex,y),sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "aaaaaand it didn't work :(\n",
    "let's try removing the columns that aren't relevant to the model (i.e. got feature_importance 0):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "old pipe\n",
      "[0.80690803 0.83104453 0.78935185 0.78640572 0.81937672]\n",
      "0.8066173715103119\n",
      "[0.85588622 0.82058862 0.84795283 0.84300688 0.83635436]\n",
      "0.8407577827290169\n",
      "new pipe\n",
      "[0.80690803 0.83104453 0.78219697 0.76788721 0.83262667]\n",
      "0.8041326807756743\n",
      "[0.85264748 0.8229142  0.84752702 0.84159843 0.8335295 ]\n",
      "0.8396433282977915\n"
     ]
    }
   ],
   "source": [
    "cols_to_drop = [x[1] for x in feature_importances if x[0]==0]\n",
    "print('old pipe',*cross_val_score_regular(best_clf,X,y),sep='\\n')\n",
    "pipe_filtered = Pipeline([('pre_processing',Preprocessor(cols_to_drop=cols_to_drop)),\n",
    "                    ('classifier', best_clf['classifier'])])\n",
    "print('new pipe',*cross_val_score_regular(pipe_filtered,X,y),sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.16241511702537537, 'Sex'],\n",
       "       [0.025204453617334366, 'Age'],\n",
       "       [0.02946314960718155, 'Parch'],\n",
       "       [0.03445915877819061, 'Fare'],\n",
       "       [0.05228187143802643, 'SibSp^2'],\n",
       "       [0.046301644295454025, 'Pclass_1'],\n",
       "       [0.05975673347711563, 'Pclass_2'],\n",
       "       [0.06751319020986557, 'Pclass_3'],\n",
       "       [0.032133691012859344, 'Title_Master.'],\n",
       "       [0.14274349808692932, 'Title_Miss.'],\n",
       "       [0.15452584624290466, 'Title_Mr.'],\n",
       "       [0.05793631449341774, 'Title_Mrs.'],\n",
       "       [0.0, 'Title_Rare'],\n",
       "       [0.025895817205309868, 'Embarked_C'],\n",
       "       [0.03161083906888962, 'Embarked_S'],\n",
       "       [0.05554375424981117, 'Cabin_Deck_-1'],\n",
       "       [0.0, 'Cabin_Deck_A'],\n",
       "       [0.0, 'Cabin_Deck_C'],\n",
       "       [0.0, 'Cabin_Deck_D'],\n",
       "       [0.022214891389012337, 'Cabin_Deck_E']], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.c_[pipe_filtered['classifier'].feature_importances_,\n",
    "      pipe_filtered['pre_processing'].get_features()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in addition, let's try creating the clf again, but incorporating the skewness of the class.\n",
    "scale_pos_weight=negatives/positives."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'objective': 'reg:squarederror',\n",
       " 'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 1,\n",
       " 'colsample_bynode': 1,\n",
       " 'colsample_bytree': 0.8162398049146752,\n",
       " 'gamma': 1,\n",
       " 'gpu_id': -1,\n",
       " 'importance_type': 'gain',\n",
       " 'interaction_constraints': '',\n",
       " 'learning_rate': 0.0592178042115199,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 3,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': nan,\n",
       " 'monotone_constraints': '()',\n",
       " 'n_estimators': 500,\n",
       " 'n_jobs': -1,\n",
       " 'num_parallel_tree': 1,\n",
       " 'random_state': 42,\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 0.6072234762979684,\n",
       " 'subsample': 0.8025175231674534,\n",
       " 'tree_method': 'exact',\n",
       " 'validate_parameters': 1,\n",
       " 'verbosity': None}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_params = pipe_filtered['classifier'].get_params()\n",
    "xgb_params['scale_pos_weight'] = ((y==1).sum()/(y==0).sum()) # positive cases/neg cases.\n",
    "xgb_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtered pipe:\n",
      "[0.80690803 0.83104453 0.78219697 0.76788721 0.83262667]\n",
      "0.8041326807756743\n",
      "[0.85264748 0.8229142  0.84752702 0.84159843 0.8335295 ]\n",
      "0.8396433282977915\n",
      "skewed pipe\n",
      "[0.72752809 0.78672493 0.78282828 0.7422138  0.74549502]\n",
      "0.7569580245248044\n",
      "[0.79243858 0.79276705 0.81424828 0.8119227  0.80018048]\n",
      "0.802311415978315\n"
     ]
    }
   ],
   "source": [
    "old_clf = best_clf['classifier']\n",
    "skew_pipe = Pipeline([('pre_processing',pipe_filtered['pre_processing']),\n",
    "                    ('classifier', xgb.XGBClassifier(**xgb_params))])\n",
    "print('filtered pipe:',*cross_val_score_regular(pipe_filtered,X,y),sep='\\n')\n",
    "print('skewed pipe',*cross_val_score_regular(skew_pipe,X,y),sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "didn't work. reverting back to filtered pipe.\n",
    "finally, we should try dealing with the fact that the data is skewed\n",
    "by stratifying the initial split maybe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_strat,X_test_strat,y_strat,y_test_strat = train_test_split(data.drop('Survived',axis=1),\n",
    "                                        data['Survived'], test_size=0.2,\n",
    "                                        random_state=42,stratify=data['Survived'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.83295455, 0.81022727, 0.82340648, 0.81191223, 0.82491582]),\n",
       " 0.8206832694763729,\n",
       " array([0.82902153, 0.84584475, 0.8419584 , 0.83912891, 0.84694805]),\n",
       " 0.8405803263200525)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score_regular(pipe_filtered,X_strat,y_strat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "We didn't change the model, only now we're looking at a more realistic test case.\n",
    "now, it seems we overfit the training set. we need to reduce the expressibility of the model.\n",
    "let's reduce the dimension of the model, by further removing features below a certain\n",
    "threshold:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtered_pipe_1\n",
      "[0.80690803 0.83104453 0.78219697 0.76788721 0.83262667]\n",
      "0.8041326807756743\n",
      "[0.85264748 0.8229142  0.84752702 0.84159843 0.8335295 ]\n",
      "0.8396433282977915\n",
      "pipe_filtered_2\n",
      "[0.80690803 0.81252601 0.77651515 0.77714646 0.83262667]\n",
      "0.8011444652874588\n",
      "[0.83678229 0.82523978 0.85034392 0.84392401 0.84227872]\n",
      "0.8397137451785561\n"
     ]
    }
   ],
   "source": [
    "cols_to_drop_2 = [x[1] for x in np.c_[pipe_filtered['classifier'].feature_importances_,\n",
    "                                   pipe_filtered['pre_processing'].get_features()] if x[0]<0.01]\n",
    "pipe_filtered_2 = Pipeline([('pre_processing',Preprocessor(cols_to_drop=cols_to_drop+cols_to_drop_2)),\n",
    "                    ('classifier', best_clf['classifier'])])\n",
    "pipe_filtered_2\n",
    "print('filtered_pipe_1',*cross_val_score_regular(pipe_filtered,X,y),sep='\\n')\n",
    "print('pipe_filtered_2',*cross_val_score_regular(pipe_filtered_2,X,y),sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I guess good enough for now.\n",
    "### visualize roc_auc score\n",
    "taken from [here](https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc_crossval.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOydd3xVRfbAv/N6SU+oSSAkVEORJqAo2Aui7rqW1UVR0bWXtW5xddV117bqKi5iX0Ww/ERcxYodCwiChN5CEtJ7ef3d+f0xL8lLTyAV7vfzucktc++ce9+9c2bOnDkjpJTo6Ojo6By+GHpaAB0dHR2dnkVXBDo6OjqHOboi0NHR0TnM0RWBjo6OzmGOrgh0dHR0DnN0RaCjo6NzmKMrgkMEIcRmIcTsnpajpxFCLBJC3N3Neb4shHigO/PsKoQQFwshPjnAcw/Zd1AIIYUQw3tajq5C6OMIOh8hRCYwAAgC1cBHwPVSyuqelOtQQwgxH1ggpZzZw3K8DORIKf/Sw3LcCwyXUv6uG/J6mV5wz92FEEICI6SUu3palq5AbxF0HXOllBHAkcBE4I89LE+HEUKYDse8exL9mev0CFJKfenkBcgETgrbfhj4IGx7OvAdUA5sBGaHHYsDXgJygTLg3bBjZwIbQud9B4xvnCcwGHADcWHHJgLFgDm0fTmwNXT9j4GhYWklcB2wE9jbwv2dBWwOyfElMKaRHH8EtoSu/xJg68A93An8AngBE3AXsBuoCl3zV6G0YwAP9a2u8tD+l4EHQuuzgRzgVqAQyAMuC8svHvgfUAmsBR4Avm3ld50Z9rtlA/PD8lwIfBCS80cgLey8J0PpK4F1wLFhx+4F3gZeCx1fABwFfB/KJw94GrCEnZMOfAqUAgXAn4DTAB/gDz2PjaG00cALoevsD92jMXRsPrAaeDx0rQdC+74NHRehY4VAReh3GQtcFcrHF8rrf43fe8AYkqv2t1sHJLfwXJv9HoCjUe9tcmh7QijN6NB2s+9GM/dWDuwJXW9+6LcoBC4NS/8ysCj0XKuAr2j6XQwPrVuBR4Gs0PNfBNh7utw5qDKrpwU4FJdGH0QSsAl4MrSdCJQAZ6BaZCeHtvuFjn8AvAHEAmZgVmj/pNDLOy30kV0aysfaTJ6fA1eGyfMIsCi0fg6wC1WQmoC/AN+FpZWhjyGuuZcbGAnUhOQ2A3eErmcJkyMDSA5dYzX1BXN77mFD6Fx7aN95KOVmAC4I5T0odGw+jQpumiqCAHBfSNYzABcQGzq+LLQ4gCNQBUSzigAYgiogfhu6VjxwZFiepagC3AQsAZaFnfu7UHoTSinlE1KOKEXgD/0uBsAOTEYVjiYgBaW0bw6lj0QV6rcCttD2tLBrvdZI7neBZwEn0B9YA/w+7PkFgBtCedlpqAhORRXgMSilMCbs2dc95xbe+9tR7/2o0LkTgPhmnmtb38PfUe+zHaWIrg87t613IwBchnrXHkAV3AtRBfkpod8zIux+qoDjQsefDH8XaKgIngDeQ73fkajKxD96utw5qDKrpwU4FJfQB1EderEksAqICR27E3i1UfqPUYXiIEAjVFA1SvMf4P5G+7ZTryjCP8IFwOehdYEq4I4LbX8IXBF2DQOqcBwa2pbACa3c293Am43O3099LS4TuDrs+BnA7g7cw+VtPNsNwNmh9fm0rQjcgCnseCGqkDWiCuBRYcdabBGgWjnLWzj2MvB8o3ve1so9lAETQuv3Al+3cc831+aNUkQ/t5DuXsIUAaqfykuYQg+d/0XY88tqdI26ZwqcAOwIPS9DS8+50Xtf+w5ur/2d2ri3Fr+H0LoZpYw2ofraRAfejZ1hx8ah3u0BYftKaKjMw5V3BKq1WdsakcBw1PdUQ8MW3wxaaD33lUXvI+g6zpFSRqIKo9FAQmj/UOA8IUR57YIyOQxC1YRLpZRlzVxvKHBro/OSUTWixrwNzBBCDEbVcCTwTdh1ngy7Rinq5U4MOz+7lfsaDOyr3ZBSaqH0LZ2/L0zG9txDg7yFEJcIITaEpR9L/bNsDyVSykDYtgv1kfdD1YLD82vtvpNRZoiWyG8mDwCEELcKIbYKISpC9xBNw3tofM8jhRDvCyHyhRCVwINh6duSI5yhqII0L+z5PYtqGTSbdzhSys9RZqmFQIEQYrEQIqqdebdXzta+B6SUflQhPRZ4TIZKXmjXu1EQtu4OXa/xvoiw7bpnIZVjRylNv69+qBbkurB8Pwrt77PoiqCLkVJ+hXqRHw3tykbVgGLCFqeU8p+hY3FCiJhmLpUN/L3ReQ4p5dJm8iwHPgHOBy4CloZ9QNko00D4dexSyu/CL9HKLeWiPl4AhBAC9dHvD0uTHLY+JHROe+8h/EMfCjwHXI8yK8SgzE6iHXK2RRHKdJDUgtyNyQbSOpqJEOJYVK33fFRLLwZlbxdhyRrfx3+AbSgvlSiUrb02fWtyNL5ONqpFkBD2vKOklOmtnNPwglL+W0o5GdUvMRJl8mnzvDbkbJyupe8BIUQicA+qr+kxIYQ1tL+td+NAqPv9hRARKNNPbqM0xSgFkh4mb7RUjiF9Fl0RdA9PACcLIY5EdQrOFUKcKoQwCiFsQojZQogkKWUeynTzjBAiVghhFkIcF7rGc8DVQohpQuEUQswRQkS2kOfrwCXAuaH1WhYBfxRCpAMIIaKFEOd14F7eBOYIIU4UQphRtmovqrOvluuEEElCiDhUIfbGAd6DE1XgFIVkvQxV66ulAEgSQlg6ID8AUsog8A5wrxDCIYQYjXpeLbEEOEkIcb4QwiSEiA/9nm0RiVI4RYBJCPFXoK1adSSq47g6JNc1YcfeBwYKIW4WQliFEJFCiGmhYwVAihDCELrHPFSF4DEhRJQQwiCESBNCzGqH3AghpoZ+KzPKHFLbOV+bV2orpz8P3C+EGBH6rccLIeKbSdfi9xCqZLyM6uy+AtU3cn/ovLbejQPhDCHEzND7dD/wo5SyQYsp1AJ+DnhcCNE/lHeiEOLUg8y7R9EVQTcgpSwC/gvcHXqxzkYVkEWoGtHt1P8W81C2620oe/bNoWv8BFyJaqqXoTpo57eS7XvACKBASrkxTJblwEPAspDZIQM4vQP3sh3V+fkUqnY0F+Uq6wtL9jqqANoTWh44kHuQUm4BHkN50BSg7Lyrw5J8jvJeyhdCFLf3HsK4HmWmyQdeBZailFpzsmShbP+3okwGG1AdoG3xMUq570CZyTy0boICuA3VkqtCFTq1ihQpZRWqQ3VuSO6dwPGhw2+F/pcIIdaH1i8BLNR7cb1NyOzSDqJC+ZeFZC+hvmX7AnBEyDzybjPn/gtVafgEpdReQHX4NqCN7+FGVD/H3aEW7WXAZUKIY9vxbhwIr6NaH6WoDvuLW0h3J+rd/SH0DX2G6hTvs+gDynQ6FaEG0y2QUn7W07J0FCHEQ8BAKeWlPS2LTvciDrMBco3RWwQ6hy1CiNEhk4UQQhyFMj8s72m5dHS6G30koc7hTCTKHDQYZYZ7DFjRoxLp6PQAumlIR0dH5zBHNw3p6OjoHOb0OdNQQkKCTElJ6WkxdHR0dPoU69atK5ZSNjvwrc8pgpSUFH766aeeFkNHR0enTyGE2NfSMd00pKOjo3OYoysCHR0dncMcXRHo6OjoHOboikBHR0fnMEdXBDo6OjqHOV2pCF5EjdbMaOG4AP6NCt70C2r2Kh0dHR2dbqYrFcHLqHlUW+J0VHTMEag5UP/ThbLo6Ojo6LRAV44j+Bo132pLnI0KzSyBH1Dzog5CxRzX0dHR6dVomqRgTwXBgNZmWl9WFlpNDQDlnnIqfJXNpnN5guzebyW3IJq8oiiK8i34AwZMXi/GQJDRxyfzr6dbmz/pwOjJAWWJNIzLnhPa15wiuCq06Ojo6PQoUkqytpTy/Tu7kBLskeYWEoImNQKVVfh37iRgMxHQAmhSwyAMgKDKY2N/eSy5ZbHsL1eLP2iqvwBg9bqweaqQwoDbWkrrE+kdGD2pCJqbUq6lCHiLQ0traXR0dHS6lKKsKr57ZxfVZV5m/CqNYRMSEEIQDAYJBAJ1/30+HwVlBawuXE38ky+TMcBN2RnHMlY7HS17Ils228nIsFBSoqzzQU2SVeMjOtLIqKGCMRFrSLd/S/+zf8fAHZuwLF1K8JhjcF55UZfcV08qghwaqrYkms4PqqOjo9PjVBa7+WHFHvZvL2PqmcMYNWMAPp+XgoIC3G43waCawdOv+Vlftp5vir9hQ/kGjg0MZ9iOaOwT3uLnZ+L4OsvY4LpOpyQ1zc/Prv38eoLkhjlWhma/RMzqpQT/uApb/BDQhsCvhsL48V12fz2pCN5DTRW4DJiGmtBb7x/Q0enl+L1B9u8oo2BvJVqwFzbQtQD4a8DnBr8rtO5S61qgw5dz+6zsyR/EuJTdzJyyHfI9uN7wAxKHwUCEwUCe9PJqIJdVWilpws7xwcFM33whHy4fwW2VIzC+4AcKiLF7mJyWyZHD9jExdR/JCUX8sK+ca60Gxg1yYvu0CPPSrxDO0Zh9IZOTwdClSgC6VhEsBWYDCaja/z1ArTFtEbASNQfsLsCFmo9UR0enlyE1SXFONVlbSsjeWkphZhX9h0YyaHgMFrux7QsctABA0Ae+avDVhJaW1mtUYW9xIs1OsDiQZifS4gS7A2lovsgTovYPEJqjRUrVH+AUQeZM2InBUEMVgzEajBhNRgSCrOoAT20v4cs9ZtKq5zLam4S/JpLn9sfiqjEQyCsjMsXOsZP2c8KUfUwZk4fJqK7vD8TyVoaGzdmf2WMHYH33G8SqTWBNgcQhUFUFg9o7vfTB0ecmppkyZYrUo4/q6HQt1WUesreWkr2llOxtZdgjzCSPiSP5iDgGj4jBYjvIOqSU4CmH6iKoKYTqQqgpDlsvCv0vVGmEAGc/ZER/cPRDc8QjHQlojgSC9ngC1jgC1li85ih8wo4/EKC1sk1KiQgV/LXparcb7zMYDBgMBsxmM0IIXC7BR6skL75Two6dJqLMUcRZ4zCKhkpxpNjESSO3ctI/TsRur5dF0zSqa1w88k0hmsHE4hEBzA/9E/LzwWiESy+FBQvAYjm4Z9wIIcQ6KeWU5o71uTDUOjo6nU+tuae28HdX+UkaE0vymDhm/Ho4kXG2ti+iBcFVqgrvmqIGhbysLeCri9QxVzGYbEhnP6QjHunoh+ZIQDoSCMaMIjBwBkF7HAFLLD5LNAGDjWAwSDAYbL3AxoBBCGw2W4N0B4uUsHWriQ8+NLPiMzeFVZU4TTGkxcQyItXAiBF+hg51ExurERMjSbCX47z1GiL+/ByGkBLQNA2v14umabywoYrqoJHX2Iz5ltAQqtGj4a9/hZEjO03u9qIrAp1Dn+8XwlcP9bQUvQopBUW+IWS7x5LtSafQO4z+1kySbRmcZM+gX0IWoljCNyC/CXPVk7X/ZCP/PYnwu9EsEWj2eIK2OAK2eFWYW+MIWFMJRk8laFf7grY4pNEakqVpjdxgMCCEqFsMBgNGoxGDoWuj4kgJfj94vYLCQgP79xvJzjbx1dcWMnbVUOItxGa0ceLUWM6ba+LoGeWYdmbgX/U5/le/Q3q96kKBAOZTTsHQrx/BYBCv14sQgujoaJ5fU8j2IjdLFkzDnNMP/vsKXHEF/O53qkXQA+imIZ1Dn4//DNYomHZ4D0WpLveRvb2K7O2VZO+oxh5hInlkJINHOBmYYsdgRrk/+gMEAgH8AT9+n59AoGkHqxACo9HYoPDWzA6E0Vx3vPZY+HpPIiWUlBjIzDSyf7+R/ftN5OYaqKgwUFVloKpK4PEItLDxYRKo9ldR4i3BEeXl3NOtXPqraAZ6duBf9Tm+zz9HOByYTzoR83HHYYiODuUl8ZnNBDUNk8lEbGwskZGRvPr+egreWM6Vi+8hLkIpQqqrISKiy+9fNw3p6JjtYI/taSm6lVpzz77NJWRvLcFT5WdAWiT9htk5bmYsJocq+CUaeZU1Dc5VtW8zBocBa6h23hY9U5dtSjAI2dlG9uwxkZ9vJD/fQF6ekcxME9XV7bgPI1gsEkNECYX2n4jsX8gVRx/Bb8da4ZuP8f/1c2pqarCceCK2B+5HS0oiGAyihfU5CCFwOhzExMQoMxXw1aMvMn7RU4yLMmL+/ng4+WSVYTcogbbQFcGhTk0x5G3saSl6lvIsiBjQ01J0OVKTFOyrIGtzCdlbyyjOriZmkJWYZAtjTo4maoClzrxSu1it1p4W+4CREqqqBPn5RnbvNrF7t4ldu0zs2WPC7wdf0EdABgAttPixOwMMGlLDoIgiBtoL6R9fSXSkG4fdh8Puw2bxo+Hlu6LvqPRXccrgkzmiOhb/h4/jW5yNedYs7DffRHDECLz+AFuLfZhKNCwWK2azOWTSAhCIKgFV1VgKdhH9+MP0//FHUvtFYD72GBg3rkefXWN0RXCo88N/IOP/IDalpyXpWQZN6GkJOoXaDtPapaK4huytZeTvrKRwrwuL3UB8ip3B422kz4nC7lQFVF/E44HCQiMFBQYKC40UFqr/xcUGioqMlJYa8PsbnuPXAlT5y/BH7sMycA8DB/txxJXhTCgjYkARA3xFHPfpfoZmVFI0yK4c28MIAgg4wxJLvDURkbGVYFwc1ot+i3HyZHzBIJ5AgJ9y3Dy/phCb2UR8RPPePULTmPbTKk76/C0sfh/9hw3E+qe74PTT611Vewm6IjjUkRpM/B0cd1tPS9Ir8Hq9+Hw+gGbNHVJKNE0jEAigaZpq8oeMxp3Rn1brhlhfexQNbOi1eWiaVieHz+fD7/erxRukLMdLcaaHkkw3PrdGQoqd/qlO0k/qjzOmc10OuxJNg6IiA1lZRvLyjHWFfX6+Wq+oaLuwdDgkMXFeDAN2Uxr9HTVx6/n1+MGclnY0Y2OPqXPp1EpK8L76Gv4v12I599dY7z+XRIejzetLKQkGg/j9fnw+HzsrYPEPhXgCGnfPTWf2yH4tm82WLYN170K0GU4+A26/HeLiOvSMugtdEegcFvj9fkpKSqiqqqrbJ4Ro4LESvh7urdK5bogSt9tdp1zaQghBdVGA4kw3xXvdlOd5iRlko3+ak9RfxxMzyNorOmJbQkqoqBAUFipbfW6ukawsE9nZRnJyjHg8LctuNEL//kEGDNDo3z9I//71/yNjXezSfmB12Wf8Uv4LU+KncPnAE5iccDcWQ70ylFVVeJe9ge+D9zGfehoRr7xc16FbS23rKhAae9D4eVqtVgq8JhZ9n8eeYhe3njKSsyckYjC08dzPOQdWrYKLL4bZszv87LoTXRHodAs+T4CszaV1rnQ+n69uBGdrBGtr5aFYLi0S+ngNtR9xbWEeWvf7/QghsJjNva5Z3hx+j0ZxZg2Fe11YHUb6pzkZcXQ8CUMdmCzdN7GglOByCcrLDZSVqf8VFQbKyw1UViovG79f4PMJ3O6Gi8ul/jfjdFRHTIxGcnKQxMQgAwfWF/YDB2rExmo09hbNqsnijb1v8P3u7xkTPYbjBx7PnePuxGFqWLuXHg/ed5bje+stTMccQ8TixRj69687XlvDF0JQ7YcVWysodWsYjSoqqBD1YxNKa8rYkF3B9cen8dy0IVhNLXSLb90KixbBgw+C0wk2Gyxe3CfeN10R6LQLKSUVFRV4PJ4DOn//lko2fJhP9CAz6kMT7fw+RH3nWwvpZSN/9uaO1efn7ZjgPYTRLOgXMvc4ojvXxh8IECrMBWVl9QV7/SLq1isqmtrhO4rTKRkwQBXyiYlBkpODJCUFSE4OEhXVPnNboaeQJbuX8H3R9/x66K9ZMGIBsdamXmDS78e3ciXe15ZgGjsW55NPYhySrNw5QyY2AJvNRmRsAst+LuSV7/dxxrhBHD0yusn1AMxGA09cOJEIawvFpdcLzz4Lr72m7F2vvALXXquO9QElALoi0GkHwWCQoqIiqqqqDrjj0efzETvYyvQLOj+Wuk7L5OcbeO89O7t2mSgrU4V7e1wow7HZJDExasRsdLSqqUdHa0RHS2w2icWiFptNYrerxeGoXz8Yx6QKXwXL9i7j07xPmZM0hxePeZEIc1N3S6lp+D//HO9LL2NISsL6t7+hDUvBq5o0CCFwOBzEx8djslh5e30uT32+jqOGxbH82mNISXAemIDr18MDD0BWlir0L7oILut7YdN0RaDTKj6fj/z8fHw+H07nAX4sgMlkQnTxqFAdhdst2LnTxAcf2Pj2WyuNuyMMBoiKUgW7KuDVogp5Wbdeu9/WjugSnY0r4OKdfe+wInsFswbOYvGMxcRZm3a0SikJ/PADnhdeRFosGG64ATF+HEaLheiICGw2G2azGZPJhJTw/qY8HvvkJ4bEOXjh0qmMS2q+FdAmNTXw1FPw9ttqOzUV7r6717mFthddERyKLJyufOcBgl449cEOX8Ln81FeXk5FRQU1JUFyNtagIoUfGK4yP+buiFR5GOB2Q2mpcp8sLTVQUmKgqMhAcbGBzEwT+/cb60xiBoNkpvtdZsqVxJlKiTZWEEk1hgqpfs59reflpeuMaQFNEtBk064iEUSIICdJAydhBPkBPj4gv5lrGJAUOONYMvYM1g4eCz8LxIZdzeYX1CSjBkby4K/GcczwhIMTfuNGpQSMRrj8ctUK6OQgcd2JrggORSpy4IZ1YI1U25b6mny4S6SmaXWdZj6fr4GbpMfjwWAw4HA4yN9cRk2Zn+SxUQcsUlyinZhBPVC17OV4vZCdbcLrhWBQ4PdDTY0y31RVKRt+7VJb8LvdrZt2jEYYOjTAxIl+To/5iOjVy3E++PduuqPWCWiSj7eV8t+1BYwZYOfSqQMZEGkhKIN8U/gl7+x7gyERwzgv+UIG2xIJBho6CZjMJsxmCxaLGavFisFoYIgzgqND7rht4bAYD9zLyuerL+yPPhquuQaOOw5GjDiw6/UidEVwqGJxgrWhLdXn85GTk9PEdbE2oFf4B+JwOBpsR/WzkDIppmtlPgyoqhJs2WImI8PM5s1mduww0ZZDVGPMZoiL0+qWhIQg8fEa/fppDB4cJCUlgMWi7ObVC17Ges01CLu9a26onUgp+XJ3Bc9+n0uC08y954zmiIFOpJR8W/gtr+x6hUhTJNeNu41RkaMwmUzYbDYcDkfduAuTydQzrrJSwqefwmOPwb/+Benpav8VV3S/LF2Ergj6Krk/q7C+zaE1dfPQNI2CgoK6TjOdeqRUo1hVjdtIUZEyt9QuXq8gGFTByA5mTFl5uRo8FX4NIWDIkCAOh8RslhiNysvG6dSIjFT2+vBCPy5Ow+mUTZxRApoks9RDQZWPov1qX+SGNfTXBL/ED4e9B27WO1hcfo03fi7EH5TcfFwSU5OcFLmL+CJ7Df+X838EZIDfj/o9s4bMwmq1YrGoUBi9gqIi+Mc/4Ouv1faKFfWK4BBCVwR9jbxfYNXfoGgH9B/dfJpRp4O5YWFfWlqKx+Npd4fvj2/tpyzXDUDAqzH0yL7bGggGYds2Ez/9ZGHzZjM1NQKvVy0uV9OIk12JyQSjRvlJT/czdmyAMWP8RER0TLtIKSmo8rOlwMWW/Bo257vYUeQiIcJMYpQVBCAll7/1Ju9POpmtGSVdczNtyYmGX5TiFXkMTSnDbC3gtdwcHtq9H6vRyrCoYcwfN58z0s7AZOxlRZGUqtB/4gkVHdThgJtvVoPEDkF62dPXaZHSvfDF32Hv13DsbXDhUjC1r3PK5XJRWlraIa+fykIvk88ahDNW5WGN6P4amqbBnj0mNm82UVTUfP7BIPj9oi6GvMdTv6ht5TPfll3dbIbYWGVmSUioNbmoxW6XCKFq6wdjmbDZJGlpgQ73Kdb4gmwrcLG5oIYt+S62FNQQ0CB9gIMjBjqZf9QARvd3EBU2a1hgwwbcJh9H3PIbxEHWrqWUrS6+gI98Tz457hyyXdnkuHLIceew372fSFMkyY5k+kcNIy12ChfFX8TI+JHEOXpnqAVAzRT2t7/B2rVqe+ZM+NOfIGxA2qGGrgh6O9WF8PUjsOltmHY1nPl4fSdwC/h8PkpKSgiGjM9ut/uAZmyyRZpwxHTeYCYp4ccfLfzvf3bKy9uWJT/fiMvVOTbhxMQgkyf7mDjRT3y8hs0msVrrfd1NveRLCGiSvSVuNofV9vMqfYzoZ+eIAQ5OGhnLjccmMijK0urv6X19KdYLL2igBDRNq4ulFP6/pRhK4eEWjEYjARkg35tPtjubbFc2WTVZZNVkkVuTSz97P1KjUkmJSuGEQSeQFpNGakwqEZaILp9MptMxmdQo4ZgYFR/olFP6zMCwA6WXvP46zZLxDnzwBxh/AVy3BiL6tXmKz+dj97ZMAj5ZZ2c1GIy4PRrga3fWWrC+cNhW6GLJD8UUZUagBQ/sg/B7jOxeG09FgQ01+rc95hANZ4yP+CE1RPXzNP8tConRJDGaNQxGicmiYTRrdf+NZg2zVcMWEaAI+KQSqGyn0FKStmM9qTs3ILp4AicJePwald4gNpNgkN3EaJuJy+wmoqxGRIWAMK9Id2sXCwQI7tuH4+ST6zzANE3DaDRiNBoxmUxN/teGpRZC4NW8ZFZlklmZyd6Kveyu2M3eir3kVeeRFJlEWkwaw6KHMWfwHNJi0hgaNRSbqY97hO3bB0lJyuUqIQEefRTS0iD28JjDQlcEvZmdn8KJf4Upl7cruc/n4+evd7Dhf8VYnQdnDjCaBZaQ3/+H3wZZuWgEVv/Be54MSAgy4+Ryho5wI0TrhaszMkhkTAddagBlJDdyMFOlRG/bxJD3liKCAQqOPQWtG0I5x1qNTI6yYDMfvBnOevFFBACfy0V0dDRxcXGYGjV7Kn2V7Cnfw57iPewu382eij3sKd9DqaeUoVFDSY1OJTUmlbPSziI1OpUhkUMwG/tmSOsW8fvhpZfgxRfhuutg3jy1f0qzE3kdsuiKoLdjbH18vqZp+Hw+3G43ezLy+GVlMUdflEx88sEX2poGS5Y4ePP5GKRfY8I4QVJSKxHEWsFggAkT/Jx0kgeLxQj0/KxMzRHYtg3P8y8gCwqwXjYf8+zZJPZy00btmJDakMnBYJCAEFgNBhITE3ELNz8X/8ye8j3srqgv8Gv8NQyLHqbMONGpTB04ldToVBIjEjEaeonXTleyZQ4fWKEAACAASURBVIvqC9i9W20XFfWsPD2Irgj6MDU1NeTn5yOlpLrEz/rlRUw5Z/BBK4FAAL76yspbbznYt8+IlAHSjyvmqfssvcaW3tkEs7LwvPgiwS1bsV0yD/NppyG68Wab64RtbMuH5udQEEJgMpkoD5aT580j15tLVk0Weyv3sqdiD0EZJC06ra7APy7pONKi0xjgHIBB9G4l1yV4PCpK6Ouvq9pOUhL85S+HXSsgnEP0sz70ycwoYs/W/ZhMaiDYzu9KGX9qfwYMb39Nu6ZGDW7avNlEQYGqAUoJW7eaKSxUBURCgsZRJ++jX2o1JlNil9xLW/i//gbX37t2ZKyw27FeeCGOP/4R0Y3TN1bUVHD5msvxBBtGdQ0F0IZ2dslIKYmyRtUV9iPjRnJ66umkxaQRb4vv1XMWdCuFhXDVVZCTo5qp8+bB739PjwRU6kXoiqAPomkaq9/ZgS3CiD1K+SKOPak/Se0IAZGTY2T1agvffWdl505TiwOkkpKCnHeei6OPdfPbJVk8MXp4Z95Ch9AqyjGffBL2G2/sukyMxoN2s+wItSGRjVYjCPjx4h8P+poWY9+NddNtJCRAfDxYrfDXvx6Sg8MOBF0R9EFKSkqQmmTUzH7EJbXPDOT1wr33RrNhQ31nn9EII0cGSE/3k5ISqPPKiYnROPJIPwYDvLuplFH9HaQl9GyIAmEwIPpwUK9aAoEAXq8Xu93OwIED8QqvmjBHL8S7jm+/heHDYeBA1Qp4+GGIilKDR3QAXRH0GQKBAB6Ph6qqKiorKzvsm714cQQbNphxOCTTp/s4+mgvkyb5MO/dgiwtbZjYBcHvwC9h85fZ/H5CP/zf5nXi3XQMbdfu7s0vFJivNjhfc9MXHghSSkwmE6YYE9td29leuB2X39UJEus0S1mZig/00UcqSNyTT6rxAPHxPS1Zr0NXBH2AyspKCgoKADCbzURERADtDxvw9ddWVq60YTLBQw+VM3x4vUtm5d1/xTh8OFia1o6Kqv3MrvAx2OfswAiErsF8wgmdcp3wjtjwJbywF0JgtVpxOBxYrdYGwc4OViFYLBae3vA0n2V9RkpUCgBzU+ce1DV1GiElfPIJPPIIlJcrM9D06Wq/3lfSLLoi6OVIJKWlpXUFUkfJzTXw5JOqA/nKK6sbKIHaHOx33oEhruGQfyklf1u2nSumDcKZeoCTd3QitQV2S8fCC/fawr620G5co68dRGW1WhtEtgwfZNWVnasSydzUuVw5/souy+OwpbBQBYn75hu1PXWq8ghK7BlHh76Crgh6OX6/H7/f3+44QSUlBl5/3UFenpFAAHJzVZiGo4/2MXdu++cbXptdhS8oOXrYgc9BcLDUzjMbCATqRr42h9FoxGw21xXktdErjUZjgxDbuufMIY7LBb/9LVRUqMnjb7kFzj5bbwW0A10R9Ca0oAop4a1W2zlrcMWOw9Sv7Z9JSvj4YxvPP++kpqbhiz9Q28dV/kVUP+BmW4ELLcxTaHhlNQ9/kYPb2TDuwtYCF5cdNQBDJ39E4SEP2kIIgdPpJDo6Grvd3ukFebmnnIySDDYVbyKzIhPZrrAXB8/20u3MTdPNQZ2OwwHnngu7dsFddx3SQeI6m65WBKcBT6LG+j8P/LPR8WjgNWBISJZHgZe6WKbeS8ADG16Hs58BYHXxOfzythNBZpOkWlBisqgacnGxgccei6zzCJo6VdX+LRYV437AI7fiHD2Lmug4Plydy5wx9WagPccdy6T0xCa1phNGxDBzWOeahGpnQ6sNedBWwS6E6LSAZZ6Ah22l28gozuCX4l/IKM6g1FNKenw6YxPGckziMRhF97iPzkqaxVEDj+qWvA5pNE0NCktKgtmz1b6rr1bvst4K6BBdqQiMwELgZCAHWAu8B2wJS3NdaHsu0A/YDiyhI9HRDjUMJhh/HgAVP2xg9PGStClNg80JAUaTgR9+sPDYY5FUVwuioiRXX13N7NneBt9BlSGAefo0tP5J/JS/g7/OG9thsWrdHg8Gs9lMUlIS9i6eLUuTGnsr9rKpeBObijaxqXgTeyv2Mix6GOMSxjFj0AyuGncVw6KHHR6hFA5Fdu2C++5TYSLi4mDaNLDblXuoTofpSkVwFCpe4p7Q9jLgbBoqAglEosZPRgClwIEFsznEUPZxL9HWCEzmpi93MAiLFjlZsUIVqlOm+PjDH6qIje1c80atKUcIQWJiIraDGIHZVXb6gpoCMoqViWdT8Sa2lGwh1hbL2ISxjEsYx9y0uYyOG933I2TqqHmDa4PEBYPK/PPnPysloHPAdKUiSASyw7ZzgGmN0jyNaiXkohTCBUBzxuOrQkufJBjUKNpXhaw1zq+4Dkqb8Y2XgGMS7CrH5XLhqQ5iMDRfcL7wglICRiPMnFvE/7wb+OK15vNfWOLm4de3kRlVQUpc+wpDKSVer5dgMEh0dDTx8fEHPX3g9tLtXPjBhWiy86YDk1ISY41RhX6/cVw29jLGxo8lxtZ3Z1TTaYGMDNUK2BOqW/7mN3DDDapjWOeg6EpF0FwJ1ri6eiqwATgBSAM+Bb6hacT4xaGluWv0erK3lPLZS1uIHRh6YXOPhwHXNhtZVEpJ4I2tBIJBDAYDEfFN03z7rYXly5US+Mc/KsgxleHJiuXuk4c2m7/rZxsvXTgKY2pqs6bTYDCIz+drMEGJlJKoqChiYmKwdlLsnWp/NeMSxvHSqZ3bDWQQBt0j6FAnGIS774bsbBgyRLmETprU01IdMnSlIsgBksO2k1A1/3AuQ3UgS5QZaS8wGljThXJ1O1KTDBoew5xrx6sd/7oELv8IYpJxu934fD40TSMQCFBZWVk3oKm5wm3/fiP/+peaoWzBghrGjfOTszUUgb+F1gNCYDSIBsc1TcPr9aJpGmazmdjY2LoavxACu92OuQuG4AuEbpfXaT+apuz+RiP88Y/www8qSFw3BgY8HOhKRbAWGAEMA/YDFwIXNUqTBZyIagUMAEZR36fQt9A02PERBJvp584yQZUZNoduze8iqGmUFBZSXl5eNzNUrQJo7ClTVGQgP99IUZGBN9904HYLjphUQ+QRuXy+E7YW1DRIL91uAmt/gpAJRtbUH6/0VbK2YC2aphEREYHNbsOECVHZ9TXqPRV986fV6QGqqlRICKtVTRcJcNRRatHpdLpSEQSA64GPUR5ELwKbgatDxxcB9wMvA5tQldo7geIulKnrKNsL/7cAhp/Y9FhxMlSOhIxVAASGHU92cTWaMYDT6WzRrCEl/Pe/DpYtczTYn5gYpHTULyzfRN2E5bPSlKun9Hio+eOfQGqI0DR7pvR0DP36Uemt5I6f7iDGFkOcMw6Dt/s9LE4c0szz0dEJ56uv1Ojg4mKwWGD+fOjX9jStOgdOV48jWBlawlkUtp4LnNLFMnQfkQPggleb7t9YBKvz4IL51NTUkJubi9VqxdpGyIglS5QSMBhg9Gg/CQkaAwZozJ3r5o+fa1xzTBLpA+s7yqTfj+vev2Ho3x/7XXciwloWLr+Lv/x0F1MGTuHumXfrNnWd3kdpqZor+JNP1Pb48apfQFcCXY4+srgbqVUCNputTQ+c1193sGSJUgJ33FHFrFmt+/DLYBD3g/8Aswn7nXc0UAK+oI97f76XlOgU/nzMn3UloNP7WLlSKYHKSjVJzPXXw/nn6+MCugldEXQhfr9feeT4/QQCgbqWQGtKQErVEqhVArfd1rYSAPA89zyyugrH3//eZIKVxzc/TqQlkgdnPah31Or0Tr79VimBo45SHkGDB/e0RIcVuiLoIr5bsYO9GwsB8Hs0nHFmrNaEViOIaho8+6yT996zYzDALbdUcfzx7RvNG9yxA9vvLkaaTHgbxfLZVbWLh497GItZn/xEp5egaSpEdG3U29tvV3MGzJmjh4foAXRF0AV4PB6ythYzeEwkCcnKhu+IMbeqBAIBePzxSD7/3IrJBHfeWcnMme2PtCGlxO3xYPH7iYyMxOFw1HkjWTIsmJuZb0BHp0fIyoIHHlCeQa++CiYTxMbCmWf2tGSHLboi6GR8Ph+5ubkYhCBmgJ34IY42zykuNvDww5Fs2mTGZpPc/ZdyJk7009oAXCE10DRk7eQqwSAJ8XHEDR3StA9Ar2Dp9AaCQViyBBYtUqEi4uKUUkhN7WnJDnt0RdCJSCAvLy9UE29fJ9fatRYefTSSykpBbKzGXeNeYejtT1DZRvP4YQniOTUEWwKakCxYcxN7s5qeZzKYcJr1Yfg6PciOHSo8xLZtavvMM+EPf1BzB+v0OLoi6Eyk6iB2ONpuBUipPINee02lnTjRz+23V2L7714MN9yA9VfntHr+gje2c8usJOJtVfxt2984Le00Vky4plNuQ0enU3nlFXjmGdUiGDhQBYmbMaOnpdIJQ1cEnYhENojX02I6CYsXO3n3XdUpfMklNZx3nhuDAdwdyK/KU8Gz+/7JzOSZXD3+6rZP0NHpCaKiVOfw+ecrt9B2VJR0uhddEXQiUso2J1LRNPjzgxa+XmXBYAxw7hUFlI36hqc2KBUwrSiTClHBtvWtu3kWUcbLOeuZmJjObVNu08cG6PQeXC7YuhUmT1bb55wD6ekwcmTPyqXTIroi6ETaUgQZGSZee83J/60K4LD5mX5uJsWxJawtfJR4qT6aFE8++dUV/FCk5hduKYSrI1oyPfEo7pxxi64EdHoPP/4If/87lJTAm2+qSeOF0JVAL6cjisAJ1LSZ6jBGarLORTQ7387mlVFED7AiJaxfb2HTJuXCabR4eeJhD8fPSMAbjOQ3X5p5/cS/A+De9m9ISmLesafhdDqprq5GCIHZbK6butHlcjFw4ECi9I42nd5CZSU88QS8957aHjkSDnJGO53uoz2K4GjUfMMRqLmFJwC/B67tQrn6HJqmIZEYjUa++srKM2+lYnGaMFvrWwhOp+Tss928WbWBMWNTgaYDvCSSgN9P0sCBRERE4Pf7qa6uxu12EwwG8fv9xMbG6kpAp/fw+efw0EOqFWCxwJVXwrx5anyATp+gPb/U46gJZEKqno3AcV0mUR+losaD1xXg4gfL2PTBEKx+jZikQgz9/ABYI/wkjislw6ZRVelp0bU/4PYQERlBREQEQN1cAbGhSKI6Or2KxYvVAjBhggoSl5LSoyLpdJz2quzsRtvBzhakr1NY6ua1tX9h789DiQBmjt/PyVe4sQwOn0BDTZ94zth4+kc0HenrKiuDNWtIuPGG7hFaR+dgOekkWLYMrr5aTR2pB4nrk7RHEWSjzEMSZcu4EdjalUL1RQrz/by+5gxsEUZ+c2YJp50QJHlcAmZr+4K8+f1+5Cef4JwyGdvw4V0srY7OAZKXBx98AFdcoTqBU1PVtj55fJ+mPYrgauBJ1GT0OcAn6P0DTdiYYUfT4MgpkmvvMAAtm3KklPilMhn5NfXfW1MDK94j4al/d4e4OjodQ9Pg7bfh6aeVe2hyMpx6qjqmK4E+T3sUwSjg4kb7jgFWd744fRMpJRs3qXmEp05tO1Dc8qzlPLfjOYxCtRbiLHE4160jMCwF+7hxXSmqjk7H2bdPhYfYuFFtn3giTJ3aszLpdCrtUQRPAZPase+wJRAIsmlrDCCZMqVtReAKuPht6m+5JO0SvF4vRoMBzzXXMvDuv3S9sDo67SUQUNFBn3tOBYmLj4c774QTTuhpyXQ6mdYUwQxU30A/4A9h+6NQcxDrhNi+PUBJiYVoexlpae2P+R8IBAgGg0Rv3UqF04lj+vQulFJHp4O8+SYsXKjWzzoLbr5ZDxJ3iNKaIrCgxg6YgMiw/ZXAb7pSqD5DyW7IXsMP7yQifUdw5NDNGAwTASj3lbO2eG2zpxVkbWZijhnX5pXExsZSvnQZCdddq48Q1uldnHsufP89/O53MG1aT0uj04W0pgi+Ci0vA/u6RZq+xprnIOt7vl99D8GAgX4jiusOfVPwDcuzljM6enST0yb/VEJ6RjXW9EiCVgvOY44m8sQTu1NyHZ2mbNgAzz4LDz8MkZFgtcJTT/W0VDrdQHv6CFzAI0A6YAvbrxsKgZoRF/NT8dFo0suRF9ebdqSUjIsax+9Tf9/knEDMEqwnWEi56y69FaDT87hcyhvozTfV9quvwrW6Y+DhRHsUwRLgDeBMlCvppUBRVwrVl/h+cwIVVUHi+9UwO12NBg4Gg/h8Pmw2G4mJiXVppZRomkZFZBRmo1FXAjo9z3ffwYMPQn4+GI0wf74aI6BzWNEeRRAPvADcRL256KuuFKpXkfMTfPVws4d2lmzlj6tHU+OpYuTEzay6/5/EFbiQEpKkn7G2GIpfv6XJeb7MTCxz53a15Do6LVNRAf/6lxoMBjBmjAoPoUcJPSxpjyLwh/7nAXOAXCCpyyTqbeT/AloAjrqqwW6JZGPmj+x7cSymoI2TZhuZ8KaLwrOOxZ40GKPJSGJEEjG2mGYvax8/vjuk19Fpnm3blBKwWFR4iIsvVi0CncOS9iiCB4Bo4FbU+IEo4OauFKrXEZMMo05rsKuqspJ3F8XhKo8nxR7k3FOG43vLwlHnXEn0iBE9JKiOTiu43fWjgKdNg5tuglmzYMiQnpVLp8dpT4So94EKIAM4HpgMlHalUL2ZGncN+3L3seSNYr5cmYpBE/zmxCyk0YfBaMCuD7fX6W1IqeYJmDMHfvmlfv+8eboS0AFabxEYgfNRMYY+QimCM4E/AXZgYpdL14sIBoNszNzIrWtupSg7lszn70Hzezh2ViYTJ8SivbYCqmswOJ09LaqOTj25ufDAA7Bmjdr++GPQzZI6jWhNEbwAJANrgH+jxhLMAO4C3u160XoXuwt2c8faOzh7wDxWvnQRRruBqLF5nB9VScSHH2AfH8HAd5dj0ucN0OkNaJpyB336afB4IDoabrsNTjut7XN1DjtaUwRTgPGAhho/UAwMB/K7Qa5eRUnQy03f3sTpiWewY+n5FOQL0pyZ3L5vAb/E30rK7fNJPkEPwtXT+P1+cnJy8Hg8PS1KzxIIKK+g/v1VsDibTSkCg0F1Eusc0thsNpKSkjCbm8550hKtKQIfSgkAeIAddFwJnIYKYW1ETXf5z2bSzAaeAMwoZTOrg3l0OpuLN7OjbIfaKMtgadVGjhp0ClHb5vPpB16iyndz6VHP8NaUBaTWRDFs9uSeFVgHgJycHCIjI0lJSTm8x2gEArB7t5ovYNAgNUpY57BASklJSQk5OTkMGzas3ee1pghGA7U9SwJIC20L1CQ1bRkajcBC4GTUPAZrUdNdbglLEwM8g1IYWUD/dkvehSzcsBC/5megcyDStZ8TiCHd+ntueTYCWZLJtFO38+GZlxBdGiQuwYFBn5WpV+DxeA5fJeDxqJAQQqi5gpOT1bbuEnpYIYQgPj6eoqKOjfltTRGMOTiROArYBewJbS8DzqahIrgIeAelBAAKDzLPTmPeEfM4Luk4PN/+h6qyn7j84Vj8fjhihI8Jg+Nhgx+pwRFn9wrdpRPisFMCmgZFRVBaCv36QUKC2u9w9KxcOj3GgXwDrSmCgw00l0jDuY5zgMYhDEeiTEJfoiKcPgn8t5lrXRVauhUpJdU11azenk5WlpHBg4McP7iQGbM9DJhxFNHRMVgd7Z32WUenk6mpUVNH+nyqJaBpbZ+jo9MMXWnTaE4tyUbbJtS4hDnAqcDdKOXQmMWozuspnSlgW7jdbrJL3bz6zUiyy72UDcrEYtIwmSG+fyw2p/nwq4HqtIrRaOTII49k7NixzJ07l/Ly8rpjmzdv5oQTTmDkyJGMGDGC+++/HynrP4kPP/yQKVOmMGbMGEaPHs1tt93W5Pper5eTTjyRI9PTeWPRIqUErFZISVGdwyFmz57NTz/91OT8l19+meuvv77JfiklN954I8OHD2f8+PGsX7++2fuTUnLCCSdQWVlZt2/58uUIIdgW1hH95ZdfcuaZZzY4d/78+bz99tuA6ti/6667GDFiBGPHjuWoo47iww8/bDbPjvCPf/yD4cOHM2rUKD7++ONm02zYsIHp06dz5JFHMmXKFNbUutaGyMrKIiIigkcffbRu30knnURZWdlBy9db6UpFkINyP60lCRWeonGaj4AaVEfx18CELpSpQ1RUVPBzjmB79ijinSauPk/V/iMjnJhMektApyl2u50NGzaQkZFBXFwcC0MTu7jdbs466yzuuusuduzYwcaNG/nuu+945plnAMjIyOD666/ntddeY+vWrWRkZJCamtrk+j+vWYO/qooNb73FBWecocxBw4Yd9LzBH374ITt37mTnzp0sXryYa665ptl0K1euZMKECUSFTVCzdOlSZs6cybJly9qd3913301eXh4ZGRlkZGTwv//9j6qqqoO6hy1btrBs2TI2b97MRx99xLXXXkswGGyS7o477uCee+5hw4YN3Hfffdxxxx0Njt9yyy2cfvrpDfbNmzev7rc6FGmvIrCj5i7uCGuBEcAw1CQ3F6I6i8NZARyLahk4UKajrR3Mp0sIBAJUV1ezemMyRmFixhSNUyY4EUBEhD5oTKdtZsyYwf79+wF4/fXXOeaYYzjllFMAcDgcPP300/zzn8qR7uGHH+bPf/4zo0er+StMJhPXNgoFXVhYyO8uu4wN27Zx5G9+w24pWfXLL0ycPJlx48Zx+eWX4/V6m8jx0ksvMXLkSGbNmsXq1c1PNb5ixQouueQShBBMnz6d8vJy8vLymqRbsmQJZ599dt12dXU1q1ev5oUXXmi3InC5XDz33HM89dRTWK1WAAYMGMD555/frvNbYsWKFVx44YVYrVaGDRvG8OHDm9T2QdnQa1s0FRUVDB48uO7Yu+++S2pqKunp6Q3OOeuss1i6dOlBydebaU+1di7wKKowHwYcCdwHnNXGeQHgeuBjlAfRi8BmVChrgEWoQv8jlDeShnIxzejQHXQSLr+LgAwAENACeDweqoMav2wbgUHArOnleEtV09Biaf90lDo9R8pdH3T6NTP/Oadd6YLBIKtWreKKUEjnzZs3M3lyQzfjtLQ0qqurqaysJCMjg1tvvbXphaSEqiqw2ejfvz/PP/88jz7yCO+//z4er5fZp57KqlWrGDlyJJdccgn/+c9/uPnm+lBgeXl53HPPPaxbt47o6GiOP/54Jk5sGhRg//79JCfXN+CTkpLYv38/gwYNapBu9erVPPvss3Xb7777LqeddhojR44kLi6O9evXM2lS69OZ79q1iyFDhjRoVbTELbfcwhdffNFk/4UXXshdd93V5B6mh033WnsPjXniiSc49dRTue2229A0je+++w6AmpoaHnroIT799NMGZiGA2NhYvF4vJSUlxMfHtyl3X6M9iuBelAfQl6HtDUBKO6+/MrSEs6jR9iOhpcfYX72fOe/MwWFSnhZCCH494Nesz5aU7k8koryC8YtvxPNCDUz5B0Z93tY+QXsL7c7E7XZz5JFHkpmZyeTJkzn55JMBZVtvqT+pxX4mv1/NE1BVBU5nfVwgIUAItm/fzrBhwxgZCh196aWXsnDhwgaK4Mcff2T27Nn069cPgAsuuIAdO3Y0ySq8r6I1uUpLS4kMG5ewdOnSuvwuvPBCli5dyqRJkzp+ry3w+OOPtztte+/hP//5D48//jjnnnsub775JldccQWfffYZ99xzD7fccgsRERHNXr9///7k5uYetooggAo6d8jiCXgYGjWUFeesAGDLDzlkbyrgsw8FSDvDk7OouuvflGsa8psyLEOH9rDEOr2V2j6CiooKzjzzTBYuXMiNN95Ieno6X3/9dYO0e/bsISIigsjISNLT01m3bh0TJkxQrYCKCigogGBQjQhupvLRXMHXHO0pfJOSksjOrnfyy8nJaWAyqcVkMqFpGgaDgZKSEj7//HMyMjIQQhAMBhFC8PDDDxMfH9+kc7W0tJSEhASGDx9OVlYWVVVVDZRKc3SkRdDee3jllVd48sknATjvvPNYsGABoJTm22+/zR133EF5eTkGgwGbzVbXue7xeA7doJJSyraWF6SUF0kpf5FSjpBSPiWlXNSO87pkmTx5suxsdpXtkmctP6tu+793fyvfXbhGpo/LlWkDCuQDs16WH7y4Vq5askmu+yhTaprW6TLoHDxbtmzpaRGk0+msW1+/fr1MTk6WPp9PulwuOWzYMPnpp59KKaV0uVxyzpw58t///reUUsqNGzfKtLQ0uX3TJikzM2Vw0yb52O23S5mVJaXPV3fNL774Qs6ZM0dKKaXb7ZbJycly586dUkopL730UvnEE09IKaWcNWuWXLt2rczNzZVDhgyRxcXF0ufzyZkzZ8rrrruuidzvv/++PO2006SmafL777+XU6dObfb+pk2bVpffokWL5FVXXdXg+HHHHSe//vpr6fF4ZEpKSt1vkpmZKYcMGSLLy8ullFLefvvtcv78+dLr9UoppczNzZWvvvpqRx51EzIyMuT48eOlx+ORe/bskcOGDZOBQKBJutGjR8svvvhCSinlZ599JidNmtQkzT333CMfeeSRum1N0+TgwYOl3+8/KBm7i+a+BeAn2UK52p4WwQ3AnwEv8DrK5v9AF+qmHsXv94O3mgRjLqXFA0m2lvEr0wtYjn2RxMTEQ7dGoNPpTJw4kQkTJrBs2TLmzZvHihUruOGGG7juuusIBoPMmzevrrY5fvx4nnjsMX57wQW4PB6EEMw54wxISlKmoGaw2Wy89NJLnHfeeQQCAaZOncrVV1/dIM2gQYO49957mTFjBoMGDWLSpEnNetKcccYZrFy5kuHDh+NwOHjppZeazXPOnDl8+eWXDB8+nKVLlzaplZ977rm8/vrrHHvssbz22mtcdtlleDwezGYzzz//PNHR0QA88MAD/OUvf+GII47AZrPhdDq57777OvyMw0lPT+f888/niCOOwGQysXDhQoyhkdULFizg6quvZsqUKTz33HPcdNNNBAIBbDYbixcvbvPa69atY/r0imW+UAAAIABJREFU6Yest6CQbTcvJwI/d4Ms7WLKlCmyOf/og2F3+W7+8OUfWHHOCtyrn2X5Gw6K3D6eXjmb04ZncFPMM/DYQlJTU/VwEr2YrVu3MmbMwQ6I72GKitTYgAEDVKiIXkZeXh6XXHIJn376aU+L0q3cdNNNnHXWWZx44ok9LUq7aO5bEEKsk1I2OxarPaXav4BtwP1Aehtp+zabl2NZ/SjBiERWlp1GlXUgI6YNJWiJwuHQYwrpdDK14SHCBmeRkACJib1SCYBqYVx55ZX/3955x0dVpX38O+mFLqB0Ij0JIRAgFpqKIEUFRcVFARHRV1bg3QWxsSK6rgUVw1JeXFkVFBAQUcFCFYF1aQYITTAEAwEhoSQhZTIzz/vHmRlSJsmkTJKZOd/P534y995z733OwJznnuec83sKLCjzBiIjI93GCZQHZ1q221AKoRdQK3wPAi+50KbqwXgVWT+N0/3mgG8Ax08EEOzvQ/iNGYhIsTMJNJpykZ0NJ08qR3Du3DV5CDdYqf7ggw86NfXTk3jiiSeq2wSX4uwr7jlUcpqnUNNH/+Yyi6qD9BTIOEvusA/Ja9CB7FxfLvwRQJ1QA2HNrgIqHqvRVBiLRc0GSkqC3FyVPL5ZMzUzSKOpJpzpf3YCHgJGAGkoFVEHK1/cmIxz4BtARr1wfDIySDwXgo/BQHgHM74+ZvIwlCnJg0bjkMIicdddpyQitBPQVDPOOIJ/A8uAARTVCvIoMjMzCQgIIOlsMEF+Bjp1MmEymfD19dXicpqKIXLNCQQGQtOmFdYH0mgqC2ccwU2lF3E/8ix5ZBiVyNUVUxag9IUCAwP5/Xwogf4+dOyYi8VswU8n99CUFxH7SmCaNIGsLNUT0L0ATQ2ipP+Nn1v/HkRpAdk2275b8+auNxm0ehDDvhzGlEMLaS7qrd9igbOpoQT6+hAerrSH9GwhjbMUkKG+804uHztmP3coKYnbH3qI9h07VkyGun9/oqOjWbFiRbF2lFWG+ujRo9x8880EBgYW0dnJj3iADPX+/fu5+eab6dy5M3fffbe9Lmlpadx2223UqlWryHfkzTLUk61/h6KE52ybbd+tyTZl83zs82wbuY1tt8zm3dw6+Pv7k5Tki9HkQ8OGJurUMRIQEOAwsYJG44jg4GDif/yRhNWraRASwryFC8FkqjwZ6l9+IS8vj/j4eB566KFKs7tBgwbExcU5dD758QQZ6vHjx/PGG29w8OBBhg8fzttvK6mzoKAgXn31VYeO0JtlqG0atE+jspXl354u7iJ3xGyxYLFY8PPz4+hRNSjc+sZcFSoKCqxm6zRuQ16eCgWdOQNmMzf37MmZ7Gzw86s8GepHHiE+Pp7o6Gh+++03Nm3aRNeuXSssQ924cWN69OhR6qQIT5ChPnbsGH369AHgzjvvZPXq1QCEhobSq1cvhzMEtQy1Sj4/vdCxQQ6OuS15eUZAiXMdPuwPCGE35mA2m/H398dYveZpysvMui64ZzH6i5cuqWmhIuDri7lhQzbFx1dchjofdhnq2bOVDHVODv369asUGWpn8QQZ6sjISL766ivuvfdeVq5cWUCorji8WYb6f1Bv/jdScEygNuD4tcJNsVgs9vDP0aN+QB5hYbkYDAa7VonGDSmu0XYFWVlgsZCdm0v0gw+SdOpUxWWoS6EyZaidxRNkqBcvXsykSZOYNWsW99xzj9P5RbxVhvoz4FvgH0B+15sBXHSlUVWNxSL4AleuGDhzxhc/31yatzAC/toRaBwjAiYT2EIp118PtWurMYL9+ysmQ+20CZUnQ+0sniBD3bFjR3744QcAfv31V9atcy6BkSfLUJc0RiBAEjAR1fjbNoAGrjWrarENKB07pn7UjetfxddH/ch8fbQj0BQiJ0etDP7992vSEH5+BXIG1K1bl7i4OGbPnk1eXh6jRo1i+/btbNy4EVAJbCZNmmTPlztt2jRef/11+9u6xWLh3XffLdGMjh07kpSUxIkTJwBYsmQJffv2LVAmNjaWrVu3kpaWRl5eHitXrqxQ1Tt06EBiYiIAq1atYvTo0Zw6dYqkpCSSk5MJCwtj+/bttGvXjpSUFI4cUZlnT506xf79+4mOjiYkJITHH3+cSZMmYTSqwOvZs2dZunRpkee99957xMfHF9kKOwFQcfzly5eTm5vLyZMnOX78OD179ixS7vz584D6jl977bUiiq2OEBHOnTtH69atnf6u3InSegRDgb0op5D/tUJQISO3pd2PibTYt4+k4FWYMi/ik36Rg1dWY07uz41t06k1fz5mLvOH0Yhv/frVba6mJmCxQFoapKaqHoG/vxogDnQ8oaDMMtRz5vDwww+TlZWlZKiHlJxhrTJlqM+dO0f37t1JT0/Hx8eHOXPmcPjw4SJxfE+QoV62bBnz5s0D4L777uOxxx6z36N169akp6djNBr58ssv+eGHHwgPD9cy1DWNypKh/mbcQOo1DaPbvY9z5dgOgo6uZGrCp+w+VJ+bOhxjxN1nCA8LpUH9+vg1aUpA82aVYL3GlbhUhjo7G1JSlD4QQP360LgxeFnoUMtQu4cCaVllqJ1xb7eihOauAo8A3YA5wO8VM7X6MTZvREiPHmRyhqD0AE78dAOGEB/qNzaS2yqM2rFhhNR1wcwTjXtx4YLaQInENWmicgh7IfllqL1JgVTLUMMCIAvoAjyLWkewxJVGVTUiFi5k1uPyZR9CQ4U6obkgaKE5jcLf/5pI3I03eq0TsKFlqD0PZxyBCTUmcC/wvnUreajfzbCYLRw91xqAdu1MShLegMfGAzWlYDZDZua1/bp1lQO4/nqtEaTxSJxp6TKA54FHgd6AL+BRr8oWsXDgdDssZiGsZS4+lwHRjsArychQKqFms2r8AwNVb6CYAWGNxhNw5vXmIVTi+nGoBDXNgLddaVRVIiJcvODHuiNDyUg1knEsGV8L+If4abE5b8JkgtOnITlZffbQ+eIajSOcaenOAZ8CdVHTSXOAT1xpVFVisVgw5UFauh91rw9k1HPXs76VEFpfNwRegQhcuQK//aZyB/v4wA03QKtWuheg8RqccQQPAruAB6yf/4vKVuYRiAipV4K4mhtA7drC9ddb1ECxk8vONW7OhQt2kThCQ1U4qEGDcucOLiBDfffdXL582X7u0KFD3H777bRv377GyVB/+umnREVFERUVxS233ML+/fsd3tcTZKjj4+O56aabiI6Opnv37nZhug0bNhATE0Pnzp2JiYlh8+bN9mu8WYbaxotAD2AMMBroCcxwpVFVicVi4eRpNfbdvn0eBgMIQkCARw2DaIqjbl21KrhpU2jZUk0PrQDBwcHEx8eTkJBAgwYN7AuXaroMdVhYGD/++CMHDhxgxowZTJgwwWE5T5ChfvbZZ3n55ZeJj49n1qxZ9tXdDRs25Ouvv+bgwYN8/PHHPProo/ZrvFmGOn+Z8/n205y8zi2wWCwkJitH0LatyX7c11cPFHskublw/rwKCYEK/7RtC/XqlbsXUBw333yzXf2ypstQ33LLLdS3rqC/6aabOH36tMNyniBDbTAY7D2aK1eu2PWIunbtav8cERFBTk6O/fvUMtTwHfA9Km8xqMHj9S6zqIoRERJPq7eb9u2vOQKdntL96fxx50q/58ExB50qZzab2bRpk1vKUH/44YcMGjTI4TlPkKGeM2cOAwcOZOrUqVgsFnbu3FmkzOrVq+natavdUXmzDLWNacB9QC+U3tAiYI0rjapKzGYLSWdqAWoNgS1m6+PrMZ0er8XeaOfkKHmInBy1X6+eWhPgAmefnZ1NdHQ0SUlJbilDvWXLFj788EO2b9/u8LwnyFAvWLCA9957j/vvv5/PP/+cxx9/3C4GCMppT58+3a5QasNbZajbAbOBNqg8xVOBou7VzUlJgcxsf2oFZdGwoZLS9TH4VKp0r6aasFiUQFxa2jWRuCZNoFYtlz3SNkbgjjLUBw4cYPz48Xz77bfFNnaeIEP98ccf8/777wPwwAMPMH78+ALXDB8+nE8++YQ2bdoUuM6TZagRkeK2n0TkCRHpICJTReSLEsoWt90lIsdE5ISIPFdCuR4iYhaREaXdMyYmRiqDr8feKZve/19ZszRJwsMuyOCuh+TMmTOSmJgot7+1SY6cvVIpz9FUHYcPHy54IDVV5NAhtZ09K2IyudyG0NBQ++d9+/ZJixYtxGg0SlZWloSFhcmGDRtERCQrK0uGDBkicXFxIiKyf/9+adOmjRw7dkxERMxms7zzzjtF7r9lyxYZMmSIiIhkZ2dLixYt5Pjx4yIiMmbMGJkzZ46IiPTt21d2794tKSkp0rJlS0lNTRWj0Si9evWSiRMnFrnvqVOnpE2bNrJjx44S6xcbG2t/3sKFC2XChAkFzvfp00e2bdsmOTk50rp1a/u/SVJSkrRs2VIuX74sIiLTpk2TsWPHSm5uroiIpKSkyJIlS0p8dmkkJCRIVFSU5OTkSGJiooSFhYnJwb95x44dZcuWLSIisnHjRunWrZuIiFy6dEmioqJk1apVRa6xWCzStGlTycvLq5CNVUWR34KIAHukmHa1pPhHbeAD4BiqZ9C6jD7GF5iHSmsZDjxs/euo3JuocYiq42oqHP2ay1/PxzfnEo3qqbCB2WzG4KN7A25L/rfk+vWhdm1o3VqtDajicZ/8MtTBwcGsXbuW1157jQ4dOtC5c2d69OjhUIa6U6dOREZGcvbs2RLvn1+GunPnzvj4+JQoQ92/f/9i4/ezZs0iLS2Np59+2j6t0hE2GWpQYaHhw4cXOG+ToQ4MDLTLUEdHRzNixIgiMtSNGjUiPDycyMhIhg0bZg9flZf8MtR33XVXERlq23TaDz74gL/+9a906dKFF154gUWLFgHwz3/+kxMnTvDqq68SHR1NdHS0PXeBN8tQH0U13rZW8VPgT/n295Vy75uBmcBA6/7z1r//KFRuCpCHmqL6DbCqpJtWmgz1g10JiY5gt+8CPl7sR6/wFGbOCyYrK4v/+eo0c//UjY43eJewlrtzJD6eTnXrqmmgerDfJWgZavdQIK1MGeqzQP4USefy7Qtweym2NAPyZ4U+DcQ6KDPceq8eJdxrgnWrNC4FD+b06ZvZnXCBq5cbERx8zSEa0D0CtyI9Hd57D3r3VusALl6ECr5dahyjZag9k5IcwW0VvLej1rRw92MOMB0ouuqjIIusm6N7lAuTz3U0qvsLTaN6UzvNny531ANU2jztB9yIzZvhzTfVgHCfPipZjAfO6qhJVHS+vzvi6TLUrgx4nQZa5NtvDqQUKtMdsK1CaQgMRslef+lCu+z4+hjJMfrg42ugdh21uMzPz0/3CNyBtDTlAGwyANHR0LCh2jQaTZlwpSPYjZqCGoaadjoSNcaQn7B8nz9CjRFUiROwoWTnDYSECCaTyb6ARFPDSUxUTiAkBJ55Bu6/H44dq26rNBq3xJWOwAT8GTUbyBdYDBwCbNMaFrrw2U5z9apSFggNFcxmM0FBQdVtkqY4MjLULCCAHj3g2WfVuECTJtVrl0bj5jjjCAzAKOBGYBbQErgBpUhaGuspKkdRnAMY68T9Kp2rVw0YzcIrm0/gXycLP//TJKVl4aenkNYcLBZYuRLmzYP33webRIIXxqo1GlfgjI7CfNRU0Iet+xmo9QEeQXa2DyazIH4mJt96A68M7cSap2+lTSPXrT7VlIGkJHjiCXj7bcjKgp9+qm6LSsRdZajXrl1LVFSUfQ1BcRIT4sEy1Lt27bKvH+jSpQtr1lxT0tEy1GrK50RUQhqAS4BHiPULkJWlvoLatYQOjYLo2qoB4U3raImJ6sZkgsWL4eGHYf9+NRNo9myYNKm6LSsRd5WhvuOOO9i/fz/x8fEsXry4gOxCfjxZhjoyMpI9e/YQHx/Pd999x5NPPonJpIQotQy1Wuzly7Vpm40Ai8ssciF/XP2D5PRkktOTEYQ8sw8mE/j6CQZfJQrmqxciVT+//w6jR8P8+ZCXB/fco0JD/fpVt2Vlwp1kqGvVqmV/+bl69WqxL0KeLEMdEhJiXzmck5NT4DvQMtQQh1IbbQz8HZWd7CVXGuUK0o3pDFw9kCahamBxGHdgMIUCEBRsAYTAwEDdE6gJ1K6tcgY0bQovvQQ9e5brNkc6diq9UBnpdPSIU+XcUYZ6zZo1PP/885w/f55169Y5LOPpMtT//e9/GTduHKdOnWLJkiV2x6BlqJW0xF7gDtTA8TDAuV9DDcJkMVEnoA7f3q/ikF+ui8PfHAJAcLASXtJTR6uRhATo0EEphNavD3FxSiMoJKTct3S20a5M3FmGevjw4QwfPpxt27YxY8aMAtLMNjxdhjo2NpZDhw5x5MgRxowZw6BBg+wzCT1ZhtqZ0FBLIAv4GvgKuGo95vZkG1U6ysBgFenSjqAayMqCt96CsWPho4+uHQ8Pr5ATqC5sYwSnTp3CaDTaxwgiIiKKDN46kqEuCyXohBWgrI1vnz59+O2330hNTS1yziZDDdhlqMePH0/r1q15++23WbFiBSLitAx1afzv//6vfQA3/2YLqeWnLDLU9913H6BkqB2Fjzp16kRoaCgJCQn2Y54sQ+2MI1iHWui1DtgEJAIVH96vAWTlqQ5RcIj6j+3vr/MUVyk7d6opoJ9/rkTiPCgsV7duXeLi4pg9ezZ5eXmMGjWK7du32988s7OzmTRpkn2gctq0abz++uv2t3WLxcK7775b7P0BOnbsSFJSEidOnABgyZIl9O3bt0CZ2NhYtm7dSlpaGnl5eaxcudLhvU6cOGF3LPv27cNoNDp88+3QoQOJiYkArFq1itGjR3Pq1CmSkpJITk4mLCyM7du3065dO1JSUjhyRPXKTp06xf79+4mOjiYkJITHH3+cSZMmYTQqWZezZ8+ydOnSIs977733iI+PL7IVDguBiuMvX76c3NxcTp48yfHjx+npIKzYtGlTfvzxRwA2b95Mu3btADh58qR9cPjUqVMcO3aM1q1bA8rpnjt3zr7vaTgTGiqc768b8KQLbKlysnNVwx8UbMEEHisxW+O4cgXefRdscehOnWDGDLCGODyF/DLUjz76KGvXruWZZ55h4sSJmM1mHn30UYcy1FlZWRgMBoYMGVLi/fPLUJtMJnr06FGiDHWTJk3o1q2bw5k0q1ev5pNPPsHf35/g4GBWrFjhsCdhk6Fu27Yty5YtK9Ig22Soe/fubZehzsnJwd/fv4gM9UsvvUR4eDhBQUGEhoYya9asMn2/hckvQ+3n51dEhvqpp56ie/fufPDBB0yePBmTyURQUJBdhnr79u288cYb+Pv74+Pjw/z582lolSzxZhnqktiHcghVTnllqC/mXGTYl8PYNlJlifpyXBwns1uwOGEQkT0zyex8gK8m36YHi11NSooKA128qJRCn3oKRo2qFNloR9K7mspFy1C7hwJpZcpQ2/hLvs8+KAdwodwW1iCyjH6AgcBgM1cNBu0EqoImTaBtW7VO4KWXVO4AjdugZag9E2ccQf6koibUWMFq15hTtWTl+mMwQGCQGR8fnazeJYjA118rWYgWLdQ4wJtvQmgo6O/cLdEy1J5HaY7AF6gFTKsCW6qc7DzrrKEgMwaDbpQqnZQUeO012LULYmJgwQLV+JeSsFyj0VQtJTkCP1QPoFrGAqqCLNtgcZDZkyasVD8WC6xYoUTicnKgbl0YNsyjZgVpNJ5ESY5gF8oJxKPWD6xErSGw8YUL7aoSVI/AQGCQRSejqSwSE1Uv4MABtT9gAEydCg0aVK9dGo2mWJwZI2gApKHyCgtqdbHgAY7A1iOwrSPQVJDMTDUjKCtL5Qx+/nmVPlKj0dRoSgqMN0bNGEoADlr/HrL+TSjhOrchOy/AOlisHUGlUKuWcgTDhyuROC90Au4qQ21j9+7d+Pr62uWiC+PJMtRGo5HHHnuMzp0706VLF7Zu3Wq/xptlqG0DxbVQM4dqFdrcHpvERFCwDg2Vi9xcpQmU/wf32GPw4ovKKXgh7ipDDUoob/r06QwcOLDYMp4sQ/3BBx8AcPDgQTZs2MBf//pXu5yGN8tQn0VlJHvFwVaxJYA1hGyjSqsQFGxB+4Eysm8fjBwJn3wC77yjnALoAeF8uJMMNcDcuXO5//77ady4cbFlPFmG+vDhw/a1Ao0bN6ZevXr2XpU3y1B7+C9ayMrL5wg0znH1quoFrLYuJbnxRvjb36AGCvbNe2pzpd9z4sLbnSrnbjLUZ86cYc2aNWzevJndu3cXa4Mny1B36dLF7kySk5PZu3cvycnJ9OzZ06tlqD13GR1gEcjN8yMA9KwhZ9mxA/7+d5UrwM8Pxo1ToaAaKtbnbKNdmbirDPWUKVN48803S03M5Mky1OPGjePIkSN0796dVq1accsttxTQFvJkGeqSHMHFKrOiGsgxWWcMBYta4Kr9QMmYTEoo7vx5iIhQvYA2barbqhqHbYzgypUrDB06lHnz5jFp0iQiIiLYtm1bgbKOZKi7dOni9LMqU4Z6z549jBw5EoDU1FTWr1+Pn58fw4YNK1DOJkPt4+Njl6FOSEjAYDBgNpsxGAy89dZbTstQ1y5lcWFZegRlkaF+//33ASVDbUvL6efnV8Dx3HLLLXZlUvBsGWpExK22mJgYKQ9p2WnSe1lv+/7ikYukww0pcvvt2bLqx3gZsWBHue7r0VgsIkbjtf34eJGlS0XM5uqzqQQOHz5c3SZIaGio/fO+ffukRYsWYjQaJSsrS8LCwmTDhg0iIpKVlSVDhgyRuLg4ERHZv3+/tGnTRo4dOyYiImazWd55550i99+yZYsMGTJERESys7OlRYsWcvz4cRERGTNmjMyZM0dERPr27Su7d++WlJQUadmypaSmporRaJRevXrJxIkTS6zDmDFjZOXKlQ7PxcbG2p+3cOFCmTBhQoHzffr0kW3btklOTo60bt3a/m+SlJQkLVu2lMuXL4uIyLRp02Ts2LGSm5srIiIpKSmyZMmSEu0qjYSEBImKipKcnBxJTEyUsLAwMZlMRcp17NhRtmzZIiIiGzdulG7duomIyNWrVyUzM1NERH744Qfp3ftae2GxWKRp06aSl5dXIRurCke/BWCPFNOueqamqhPk5PljAEJCLBgMBh0aKsz58/DGGypb2IwZ6liXLmrTOIU7yVA7iyfLUJ8/f56BAwfi4+NDs2bNWLJkif3eni5DXe1v+GXdKqtH8M69S6XDDSnywAOZsmb7QXlgwc5y3dfjsFhEvvhCpE8fkZgYkb59RdLSqtsqp6gJPQJPJyUlRfr371/dZlQ5kyZNko0bN1a3GU6jewROkpvnjwiYfPI4l2mqbnNqBqdPK3kI20KkPn3guee0PITGjpah9ky81hFk5PqSY7Kw949LXNqfxm2dmlS3SdWHCCxbpkTicnOhXj149lm48069LkBTBC1D7Xl4rSPINQUQ4OvDvTH1+csjnRzOLvAaDAY4cUI5gbvuUiJx9epVt1UajaaK8FpHkGPNRRASYvHOpDR5eXDhAtgc4JQpcMcdcOut1WuXRqOpcrywBVTY1hGEhJhLXUTjcRw+DI88As88A0ajOlanjnYCGo2X4rWOINdkS0rjRT2CnByYM0cphP72m0ogc+5cdVul0WiqGVe3gHcBx4ATwHMOzo8CDli3nUCVTVLPMSmdoeBgL8lXvGcPPPQQLF2q9kePVgPEOnl8pXLu3DlGjhxJmzZtCA8PZ/DgwSxatKiIJLNGU5Nw5RiBLzAPuBM4DexGZTo7nK/MSaAvcAkYBCwCYl1okx3bGEFwsMVzF4nYmDsXPv5YfW7bVslDhIdXr00eiIgwfPhwxowZY1fijI+P5+uvv65myzSaknHlq3BPVE8gETACy4F7C5XZiXICAD8DzV1ojx2LxaJCQ4ZrK4s9mjZtlEjcU0/BkiXe4wS6dy9++yJfgr0vvii5rJNs2bIFf3//Aqt7o6Oj6d27N5mZmYwYMYKOHTsyatQou07QrFmz6NGjB5GRkUyYMMF+vF+/fkyfPp2ePXvSvn17fvrpJ0Cpmk6dOpXOnTsTFRXF3LlzAbXytW/fvsTExDBw4EDOnj1b0W9P40W40hE0A5Lz7Z+2HiuOx4HiUhRNAPZYtwojIvbB4tBQ8bzQ0KVL8OOP1/YHDVKN3fjxNVYp1BNISEgoIjVt45dffmHOnDkcPnyYxMREe16AP//5z+zevZuEhASys7P55ptv7NeYTCZ27drFnDlzeOWVVwBYtGgRJ0+e5JdffuHAgQOMGjWKvLw8nnnmGVatWsXevXsZN24cL774ousrrPEYXBkTcfSaXZxc4m0oR9CrmPOLrFtJ93Aae48A5Qg8pkcgorKFvf22yhv82WcQFqbWCXjjOgkHqRodct99anMhPXv2pHlz1eG1yVT36tWLLVu28NZbb5GVlcXFixeJiIjg7rvvtpqlbIqJiSEpKQmAjRs38tRTT9nDmQ0aNLBn+bJJXpvNZpo08eIFkpoy40pHcBpokW+/OZDioFwU8C/UGEGaC+2xY7FYCqwj8AhH8Mcf8I9/wPbtar9nzxqZLMaTiYiIKDbXb2C+fwtfX19MJhM5OTk8/fTT7NmzhxYtWjBz5kxycnKKXGMrD47zGogIERER/Oc//6nsKmm8BFfGRHYD7YAwIAAYiRoszk9L4AvgUaBotgwXkT805PYLyiwWFfZ54AHlBGrVUoPB8+Z5Zy+gGrn99tvJzc21574FlQz+x/xhunzYGv2GDRuSmZlZrBPJz4ABA1i4cKHdMVy8eJEOHTpw4cIFuyPIy8vj0KFDFa2OxotwZQtoAv4MfA8cAT4HDgFPWTeAvwHXAfOBeCppDKA0cnIsmC2++Bos+PuXP0NUjWDuXHj9dRUK6tcPVq6Ee+7RGkHVgMFgYM2aNWzYsIE2bdoQERHBzJkzi5UvqVevHk888QSdO3dm2LBh9OjRo9RnjB8/npYtWxIVFUXJpYuEAAAY40lEQVSXLl347LPPCAgIYNWqVUyfPp0uXboQHR1tT7+o0TiDwTZLwV3o3r277HE29puPizkXGfblMLaN3EZy8lX6RWdSp1YWKzeZad26tftOIU1OhokTYdIkJRHhxQ7gyJEjdOrUqbrN0GiqHUe/BYPBsFdEHE6Dc+OYSPnJzFTOL8g/D8C9QkPHj8M776iBYYAWLWDNGujf36udgEajKT9u+hpcMdLTLQAE+RsRCXSP0JDRCIsXw7//DWYzdOoEgwerc96mlaTRaCoVr3YEwf5GfHyCa74jOHgQXn0VEhPV/oMPqvEAjUajqQS80hHYQkOB/nk1OyyUnQ0LFihNIBGlCzRjBnTtWt2WaTQaD8IrHUFGhm2MwFizHcEXX6hFYT4+MGYMTJgAAQHVbZVGo/EwvNoRBPsba14uApFrg74PPghHjqjcAR07Vq9dGo3GY6nBr8Ou4+pV9TeopoWGtm6FUaPg8mW17++vkslrJ6DRaFxIDWoFq46LF9Ubd63AnJrRI7h4EZ57TuUK/vVXtShMo9FoqgivdATnz6tq1wvNrN4egQisXw8jRsDGjRAcDM8+C48/Xn02aSqMwWDg0Ucfte+bTCYaNWrk8uQ0vr6+REdHExkZyd13381lW88SOH36NPfeey/t2rWjTZs2TJ48GaMtTSmOE+r8+mtR1Zfs7Gz69u2L2Wy2H1uzZg0Gg4GjR4/ajyUlJREZGVng2pkzZzJ79uwyPa+sfPfdd3To0IG2bdvyxhtvOCzz3nvvERERQWRkJA8//LBd6uPy5ct2qfBOnTpVinaTM/aUVs5sNtO1a9cC/3+MRiN9+vSxS41UFK9zBBaLhdRU1QuoH1KNjuDcOZg8WekCpafDTTfBihVqXKAmhas0ZSY0NNQuKw2wYcMGmjUrSYG9cggODiY+Pp6EhAQaNGjAvHnzAKWtdd999zFs2DCOHz/Or7/+SmZmpl2q2pZQp1+/fvz2228cPnyY119/nT/++KPIMxYvXsx9991XoCe9bNkyevXqZU/GUxpleV5ZMJvNTJw4kW+//ZbDhw+zbNkyDh8+XKDMmTNniIuLY8+ePSQkJGA2m+12T548mbvuuoujR4+yf//+Ulepb926lbFjx1bIHmfKvf/++0VsCQgI4I477mDFihWlfS1O4XWDxSJidwT1QjKrT1oiJQV27oTateEvf4GhQ/XK4EqkDPlkyoSz6iaDBg1i3bp1jBgxgmXLlvHwww/bk8ssXbqUuLg4jEYjsbGxzJ8/H19fX4YNG0ZycjI5OTlMnjyZCRMmkJSUxKBBg+jVqxc7d+6kWbNmrF27luDg4BKff/PNN3PgwAEANm/eTFBQEI899higeg7vvfceYWFhvPLKK/z8888OE+o44tNPP+Wzzz6z72dmZrJjxw62bNnCPffcw8yZM0v9bopL4FNRdu3aRdu2bbnxxhsBGDlyJGvXriW8UCImk8lEdnY2/v7+ZGVl0bRpU9LT09m2bRsfffQRoBragArO0HPWnpLKnT59mnXr1vHiiy/y7rvvFrhu2LBhPP/884waNapCdoIX9QhMeXnk5WTy07zJpCX/gY+PUDvoatX2CC5duva5WzfVG1i1Cu6+WzsBD2PkyJEsX76cnJwcDhw4QGysysB65MgRVqxYwY4dO4iPj8fX15dPP/0UUG/be/fuZc+ePcTFxZGWplTZjx8/zsSJEzl06BD16tVj9erVJT7bbDazadMm7rnnHgAOHTpUJGFOnTp1aNmyJSdOnCgxoU5+jEYjiYmJtG7d2n7syy+/5K677qJ9+/Y0aNCAffv2lXofZ58H0Lt3b6Kjo4tsGzduLFL2zJkztGhxTfm+efPmnDlzpkCZZs2aMXXqVFq2bEmTJk2oW7cuAwYMIDExkUaNGvHYY4/RtWtXxo8fz1XbrJJCxMbGEh0dzfjx4/nqq6/sNn3//fdltqe0clOmTOGtt95y2E5FRkaye/duhzaWFa/pEWReTsXPkstFU2vEx5/gEANNmtaumlXFZrNKGv9//wfvvw82lUnrD1VT+ZRDl7BSiYqKIikpiWXLljHYJgUCbNq0ib1799qVRrOzs2ncuDEAcXFxrFmzBoDk5GSOHz/ODTfcQFhYmP2NOX+SmsJkZ2fbk97ExMTYE9U4ymFQ0vHiSE1NpV69egWOLVu2jClTpgDK+S1btoxu3boVe9+y/t5svShncCSgWfh5ly5dYu3atZw8eZJ69erxwAMPsHTpUjp27Mi+ffuYO3cusbGxTJ48mTfeeINXX321yD3/+9//Aio09NFHH9l7EeWxp6Ry33zzDY0bNyYmJoatW7cWKePr60tAQAAZGRnUrl3boQ3O4jWOAFRqs0Y9nsE30ExDv7PUqh3i+h7Br7/CrFlgG0jbvfuaI9B4NPfccw9Tp05l69at9rd7EWHMmDH84x//KFB269atbNy4kf/85z+EhITQr18/+yBm4aQ2trGHwtjGCK5cucLQoUOZN28ekyZNIiIiokgvIj09neTkZNq0acP58+edyoUQHBxcIHFOWloamzdvJiEhAYPBgNlsxmAw8NZbb3HddddxKX8PGJU7ISwsjObNmzv1PFA9goyMjCLHZ8+eTf/+/Qsca968OcnJ17Ljnj59uogE+MaNGwkLC6NRo0aAygK3c+dO+vfvT/Pmze09txEjRpQ4uOsMzthTUrkdO3bw1VdfsX79enJyckhPT+eRRx5h6dKl9rK5ubkEBQVVyE5A/cd0py0mJkbKw2+/J8itH4bLypU50qZlpozq/rMcW7BAsrKyynW/UsnNFZk3T6RHD5GYGJGhQ0X+8x/XPEsjIiKHDx+ubhNERCQ0NFRERJKTk2XOnDkiIrJlyxYZMmSIHDp0SNq2bSt//PGHiIikpaVJUlKSfPnllzJ06FARETly5IgEBgbKli1b5OTJkxIREWG/99tvvy0vv/xyic8VEdm3b5+0aNFCjEajWCwWiYmJkY8//lhEREwmk4wfP17+8pe/iIiIxWKRnj17yqJFi+zX79q1S7Zu3VrkGc2bN5fs7GwREVm4cKFMmDChwPk+ffrItm3bREQkJiZGNm7caK9nu3bt5MSJE2V6XlnIy8uTsLAwSUxMlNzcXImKipKEhIQCZX7++WcJDw+Xq1evisVikdGjR0tcXJyIiPTq1UuOHj0qIiIvv/yyTJ061eX2OFvO9v8nP6mpqdKxY0eHz3b0WwD2SDHtqteMEdg4d04w5VpoEaJmKLikR5CYCH/6k1ILFYGHHlIzgm66qfKfpamxNG/enMmTJxc4Fh4ezmuvvcaAAQOIiorizjvv5OzZs9x1112YTCaioqKYMWMGN1Xw/0rXrl3p0qULy5cvtyfMWblyJe3ataN9+/YEBQXx+uuvA2VLqDNgwAC2W9OhLlu2jOHDhxc4f//999sHkz/55BNee+01oqOjuf3223n55Zdp06ZNmRP4OIufnx///Oc/GThwIJ06deLBBx8kIiICgMGDB5OSkkJsbCwjRoygW7dudO7cGYvFwoQJEwCYO3cuo0aNIioqivj4eF544QWHz7GNERTeCo8RlGRPfptKK1ccW7ZsKRB2rBDFeYiaulW0R/CXZ9Kl9Q3pEjfoQzm2YIHk5uaW634lcuWKyJ13itx/v0h8fOXfX+OQmtIj8GT27dsnjzzySHWboRGR4cOH23swhSlrj8CrxggATh7Pwy8wkIYhGUAl5iLYuxc6d1aicHXqqJzBrVppkTiNR9G1a1duu+02zGZzzViV76UYjUaGDRtGhw4dKuV+XhcaOvO7Bf8gH6sjqITQUHo6vPIKPPmkCgXZaNdOOwGNRzJu3DjtBKqZgIAARo8eXWn38ypHEJJbn0tX/PD196FhSLpV6LMCPYLNm5U8xNdfq0a/Vq3KM1aj0WiqCK8KDTU+152Tvv4EBQm1AnIRQzkdQVoavPmmcgSgEsW89JIKBWk0Go2b4VWOoG5Kd/yCfGjUyIwBEAxldwRnzsCjj6qQUEgIPPMM3H+/1gfSaDRui9c4AnOe4JvaBr8AHxo2NCEIBp9y9AaaNoWICCUJ8cILcMMNlW+sRqPRVCFe4whMRiE9Rw1wNW5sgVwnB4otFpUf4KabVOjHYFBhoeBgrQ+k0Wg8Aq9xBADpWYGA0KiRBU6DT2kN+cmT8OqrcOAAREfDBx+oxj8kpErs1Wg0mqrAqxxBZpbSbLn+eguSLBiK6xGYTPDJJ6rhz8uDhg1V3mDdA9BoNB6IVzmCjKwg8IGGDS0IxfQIjh5VInG2bEn33gtTpqi8ARq3Ijk5mdzc3Eq7X2BgYAG54Mpg3LhxdpXJhIQEp6+7fPkyn332GU8//bTD8zNnzqRWrVpMnTrVqfuVtbzGs/CqqS4Z2apH0KiRBZCiYwQZGTBhgnICTZvC/PkwY4Z2Am5Kbm4uISEhlbaV1amUlsEKYOzYsXz33Xdlrtvly5eZP39+ma/TaBzhNY5ABDKtjqBhQzM4WkxWuzY88YQSjFuxAnr2rAZLNd5Enz59aNCgQYllrl69ypAhQ+jSpQuRkZGsWLGC5557jt9++43o6GimTZsGwN///nc6dOhA//79OXbsWKnPLqn80qVL6dmzJ9HR0Tz55JOYzWamT59ewPnMnDmTd955pxy11tQ0vCY0dOWKL2azD7VqWQgOhnQEn7w8NQMoMhKGDFEF8yUd12jKQ2xsLLm5uWRmZnLx4kV7Upk333yTgQMHlvl+3333HU2bNmXdunUAXLlyhdjYWBISEoiPjwdg7969LF++nF9++QWTyUS3bt1KzAJWUvn8WdT8/f15+umn+fTTTxk5ciRTpkyxh6M+//zzcvVmNDUPr3EEaWmqqg0bmhERgs+epcHcueDrC5s2wZ13am0gTaXgbAYrZ+ncuTNTp05l+vTpDB06lN69exdJ+vLTTz8xfPhwQqwz2u4pJftdSeWLy6I2evRozp8/T0pKChcuXKB+/fq0bNmyQnXT1Axc7QjuAt4HfIF/AYVT/his5wcDWcBYoPSkp+UgNVVVtVGdbILfeIOQHTvwrV0bbrlFjQNoJ6CpobRv3569e/eyfv16nn/+eQYMGOBQcKysq+SLKy/FZFEDlblr1apVnDt3jpEjR5bpeZqaiyvHCHyBecAgIBx42Po3P4OAdtZtArDAVcakpfoSYMyi1fbVhP74IwY/P3IHD4F//1sphWo0lUy/fv0q3BsASElJISQkhEceeYSpU6eyb98+ateuXSCFY58+fVizZg3Z2dlkZGTw9ddfl3jPksrfcccdrFq1ivPnzwMqxeSpU6cAlZd4+fLlrFq1ihEjRlS4bpqagSt7BD2BE0CidX85cC9wOF+Ze4FPUOmEfwbqAU2As5VtTFqKhZDsdJpYjmBq6M+Fdu2p16+fCg1pPJLAwECysrIq9X7OYBsjKIyjMYKHH36YrVu3kpqaSvPmzXnllVd4/PHHC5Q5ePAg06ZNw8fHB39/fxYsWMB1113HrbfeSmRkJIMGDeLtt9/moYceIjo6mlatWtG7d2/79YMHD+Zf//pXgQxg3bp1K7Z8/ixqFosFf39/5s2bR6tWrYiIiCAjI4NmzZrRpEmTEp+hcR8MKnGNSxiBCg2Nt+4/CsQCf85X5htUuGi7dX8TMB3YU+heE6wb3bt3j9mzp/Dp0nn91Uss++cfzOi+if4j64CPD3UGDsSvYcMy30tTMzly5AidOnWqbjM0mmrH0W/BYDDsFZHujsq7skfgKABZ2Os4UwZgkXUr7nypvDCjPi/MqI9IR71AWKPRaPLhyjGC00D+ZZjNgZRylKlUtBPQaDSagrjSEexGDQKHAQHASOCrQmW+AkajegY3AVdwwfiAxntwYahTo3ELyvMbcGVoyIQaD/geNYNoMXAIeMp6fiGwHjV19ARq+uhjLrRH4+EEBQWRlpbGddddV7EUpBqNmyIipKWlERQUVKbrXDlY7BK6d+8u5Rks1ng+eXl5nD59mpycnOo2RaOpNoKCgmjevDn+/v4FjlfXYLFGU6X4+/sTFhZW3WZoNG6H14jOaTQajcYx2hFoNBqNl6MdgUaj0Xg5bjdYbDAYLgCnynPt9ddf3/CPP/5IrWSTajS6zt6BrrN3UME6txKRRo5OuJ0jqCB7AIej5h6MrrN3oOvsHbikzjo0pNFoNF6OdgQajUbj5XibI1hUehGPQ9fZO9B19g5cUmdvGyPQaDQaTSG8rUeg0Wg0mkJoR6DRaDRejqc6gruAYyhV0+ccnDcAcdbzB4BuVWeayyitzqNQdT0A7AS6VJ1pLqO0OtvoAZhRWfPcHWfq3A+IR6n9/lg1ZrmU0upcF/ga2I+qs7urGC8GzgMJxZyv/PZLRDxt8xWR30TkRhEJEJH9IhJeqMxgEflWRAwicpOI/LcG2O3qOt8iIvWtnwd5SZ1t5TaLyHoRGVED7HZ1neuJyGERaWndb1wD7HZ1nV8QkTetnxuJyEVr2eq2vbxbHxHpJiIJxZyv9PbLE3sEPVGeMhEwAsuBewuVuRf4BJX28megHtAE98WZOu8ELlk//4zKBufOOFNngGeA1ag3LHfHmTr/CfgC+N267+71dqbOAtRGvSnXAi6i8qG4K9tQdSiOSm+/PNERNAOS8+2fth4raxl3oqz1eRz41qUWuR5n/52Ho5IgeQLO1Lk9UB/YCuxFZQB0Z5yp8z+BTqg0tweByYClSqyrHiq9/fLEfASOUlMVniPrTBl3oiz1uQ3lCHq5zpwqwZk6zwGmo8YHPAFn6uwHxAB3AMHAf1Bvjb+61jSX4UydB6LGRG4H2gAbgJ+AdNeaVm1UevvliY7gNNAi335z1JtCWcu4E87WJwr4FzAISKsCu1yJM3XujgolADREpUU1AV+63DrX4Oz/7VTgqnXbhpoY4K6OwJk6Pwa8gWoMTwAngY7ArqowsBqo9PbLE0NDu4F2QBgQAIwEvipU5itUl9kA3ARcAc5WoY2VjTN1bomKHT+K+zYK+XGmzmFAa+u2Cnga93UC4Fyd1wK9US95IUAscKQKbaxsnKnz76geEMD1QAfUmIKnUuntlyf2CEzAn4HvAV/UVKxDwFPW8wuB9ai3wxNAFu4/3cyZOv8NuA6Yn+8ad1ZudKbOnoYzdT4CfIeaVmhB9QCLm4boDjhT51eBj1DjAwZUONCd5amXoaYAN0S9/b8M2BIQu6T90hITGo1G4+V4YmhIo9FoNGVAOwKNRqPxcrQj0Gg0Gi9HOwKNRqPxcrQj0Gg0Gi9HOwJNTcWMWi1q21qXUDazEp73EWohUjywD7i5HPf4FxBu/fxCoXM7y21ZQWzfSwJKcbNeKeWjUVMNNZpi0dNHNTWVTJSAWGWXLY6PgG9QC88GALNRK7HLS2XYVNp9P0YtDvx7CeXHotaL/NkFtmg8BN0j0LgLtYBNqLf1gzhWGm2CklSwvTH3th4fgNLc2QespPQGehvQ1vr5L9Z7JQBTrMdCgXUo/fsE4CHr8a2oRvcNlM5PPPCp9Zyt17KCgm/oHwH3oxZLvY1aSXsAeLIUG7HWySY21hPV6/jF+rcDaiXuLKt98da/oahFWbutZR19jxovwxNXFms8A1tDCipk8wBKSTQdteLyZ9RS+/xd2j+hVqD+HdWwhljLvgT0R2nvTEc17rNKePbdKGcTg1q1GYtasfpfVKKXG1HaLkOs5esWuv451Bt4tIN7L0c1yOtRDfUdwP+ghACvoJLoBAI7gB+sdXeEr/XaD637R4E+qJW4/YHXUQ7mbxTsEbwObAbGocJKu4CNqO9G46VoR6CpqWRTsCH1RzVifVDSCc1QujLn8pXZjXrb9UdpCsUDfVFx+x3WMgGoN2lHvI1yGhdQDfMdwBquNZJfoHoZ36FCR2+iwkk/laFe36KySwWiMm9ts9Z1ACoUZcuiVhelsVPYEdgcZGuUzPSGfOU/tl4jXJMkKMwA4B5gqnU/CKVD5c56RJoKoh2Bxl0YBTRCvaXnAUmoRiw/21COYgiwBNWwX0I1lg878YxpqDECG/2LKfer1Y7BwD9Qb+4l9TDyk4MKIQ1E9QyWWY8bUEl0vi/lepuDrItyQhNRjuVVYAuq19Ta+gxHGFA9hWNO2qvxAvQYgcZdqIvKtpWHyqnQykGZVtYyH6BCJt1QIaRbuRbzD0Elb3GGbcAw6zWhqEb2J6ApSuxrKapn4ChnbB7Fv5UvR4WcenOt4f8eFSKyXdPe+sziuAJMQr3Z+6O+nzPWc2PzlctAZe+y8T3K4dg07buW8AyNl6AdgcZd+BQV696D6h0cdVCmHyps8gvqrfd9VJhnLOrN+wDKMXR08pn7UIO5u1DjA/+y3ruz9Vg88CLwmoNrF1mf96mDcz+gei4bUekXsd77sPWZCcD/UXqP/RfUgPVI4C1U72QHavzAxhZUaMw2WPwqynEcsD7n1VKeofEC9PRRjUaj8XJ0j0Cj0Wi8HO0INBqNxsvRjkCj0Wi8HO0INBqNxsvRjkCj0Wi8HO0INBqNxsvRjkCj0Wi8nP8HVsK4ZALbInAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tprs = []\n",
    "aucs = []\n",
    "mean_fpr = np.linspace(0, 1, 100)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "kf = StratifiedKFold()\n",
    "lr_clf_pipe = pipe_filtered_2\n",
    "for i, (train, test) in enumerate(kf.split(X, y)):\n",
    "    lr_clf_pipe.fit(X.iloc[train], y.iloc[train])\n",
    "    viz = plot_roc_curve(lr_clf_pipe, X.iloc[test], y.iloc[test],\n",
    "                         name='ROC fold {}'.format(i),\n",
    "                         lw=1, ax=ax)\n",
    "    interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)\n",
    "    interp_tpr[0] = 0.0\n",
    "    tprs.append(interp_tpr)\n",
    "    aucs.append(viz.roc_auc)\n",
    "\n",
    "ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',\n",
    "        label='Chance', alpha=.8)\n",
    "\n",
    "mean_tpr = np.mean(tprs, axis=0)\n",
    "mean_tpr[-1] = 1.0\n",
    "mean_auc = auc(mean_fpr, mean_tpr)\n",
    "std_auc = np.std(aucs)\n",
    "ax.plot(mean_fpr, mean_tpr, color='b',\n",
    "        label=r'Mean ROC (AUC = %0.2f $\\pm$ %0.2f)' % (mean_auc, std_auc),\n",
    "        lw=2, alpha=.8)\n",
    "\n",
    "std_tpr = np.std(tprs, axis=0)\n",
    "tprs_upper = np.minimum(mean_tpr + std_tpr, 1)\n",
    "tprs_lower = np.maximum(mean_tpr - std_tpr, 0)\n",
    "ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,\n",
    "                label=r'$\\pm$ 1 std. dev.')\n",
    "\n",
    "ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],\n",
    "       title=\"Receiver operating characteristic example\")\n",
    "ax.legend(loc=\"lower right\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# the final test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score:  0.8258317025440314\n",
      "[[90 15]\n",
      " [15 58]]\n",
      "fp: \n",
      "      Pclass                                           Name     Sex   Age  \\\n",
      "434       1                      Silvey, Mr. William Baird    male  50.0   \n",
      "854       2  Carter, Mrs. Ernest Courtenay (Lilian Hughes)  female  44.0   \n",
      "396       3                            Olsson, Miss. Elina  female  31.0   \n",
      "235       3                   Harknett, Miss. Alice Phoebe  female   NaN   \n",
      "137       1                    Futrelle, Mr. Jacques Heath    male  37.0   \n",
      "578       3               Caram, Mrs. Joseph (Maria Elias)  female   NaN   \n",
      "593       3                             Bourke, Miss. Mary  female   NaN   \n",
      "852       3                        Boulos, Miss. Nourelain  female   9.0   \n",
      "807       3                Pettersson, Miss. Ellen Natalia  female  18.0   \n",
      "49        3  Arnold-Franchi, Mrs. Josef (Josefine Franchi)  female  18.0   \n",
      "254       3       Rosblom, Mrs. Viktor (Helena Wilhelmina)  female  41.0   \n",
      "745       1                   Crosby, Capt. Edward Gifford    male  70.0   \n",
      "456       1                      Millet, Mr. Francis Davis    male  65.0   \n",
      "362       3                Barbara, Mrs. (Catherine David)  female  45.0   \n",
      "772       2                              Mack, Mrs. (Mary)  female  57.0   \n",
      "\n",
      "     SibSp  Parch       Ticket     Fare Cabin Embarked  \n",
      "434      1      0        13507  55.9000   E44        S  \n",
      "854      1      0       244252  26.0000   NaN        S  \n",
      "396      0      0       350407   7.8542   NaN        S  \n",
      "235      0      0   W./C. 6609   7.5500   NaN        S  \n",
      "137      1      0       113803  53.1000  C123        S  \n",
      "578      1      0         2689  14.4583   NaN        C  \n",
      "593      0      2       364848   7.7500   NaN        Q  \n",
      "852      1      1         2678  15.2458   NaN        C  \n",
      "807      0      0       347087   7.7750   NaN        S  \n",
      "49       1      0       349237  17.8000   NaN        S  \n",
      "254      0      2       370129  20.2125   NaN        S  \n",
      "745      1      1    WE/P 5735  71.0000   B22        S  \n",
      "456      0      0        13509  26.5500   E38        S  \n",
      "362      0      1         2691  14.4542   NaN        C  \n",
      "772      0      0  S.O./P.P. 3  10.5000   E77        S  \n",
      "\n",
      "\n",
      "fn: \n",
      "      Pclass                                               Name     Sex   Age  \\\n",
      "804       3                            Hedman, Mr. Oskar Arvid    male  27.0   \n",
      "507       1      Bradley, Mr. George (\"George Arthur Brayton\")    male   NaN   \n",
      "23        1                       Sloper, Mr. William Thompson    male  28.0   \n",
      "447       1                        Seward, Mr. Frederic Kimber    male  34.0   \n",
      "737       1                             Lesurer, Mr. Gustave J    male  35.0   \n",
      "209       1                                   Blank, Mr. Henry    male  40.0   \n",
      "298       1                              Saalfeld, Mr. Adolphe    male   NaN   \n",
      "821       3                                  Lulic, Mr. Nikola    male  27.0   \n",
      "204       3                           Cohen, Mr. Gurshon \"Gus\"    male  18.0   \n",
      "677       3                            Turja, Miss. Anna Sofia  female  18.0   \n",
      "570       2                                 Harris, Mr. George    male  62.0   \n",
      "744       3                                 Stranden, Mr. Juho    male  31.0   \n",
      "109       3                                Moran, Miss. Bertha  female   NaN   \n",
      "97        1                    Greenfield, Mr. William Bertram    male  23.0   \n",
      "25        3  Asplund, Mrs. Carl Oscar (Selma Augusta Emilia...  female  38.0   \n",
      "\n",
      "     SibSp  Parch             Ticket      Fare    Cabin Embarked  \n",
      "804      0      0             347089    6.9750      NaN        S  \n",
      "507      0      0             111427   26.5500      NaN        S  \n",
      "23       0      0             113788   35.5000       A6        S  \n",
      "447      0      0             113794   26.5500      NaN        S  \n",
      "737      0      0           PC 17755  512.3292     B101        C  \n",
      "209      0      0             112277   31.0000      A31        C  \n",
      "298      0      0              19988   30.5000     C106        S  \n",
      "821      0      0             315098    8.6625      NaN        S  \n",
      "204      0      0           A/5 3540    8.0500      NaN        S  \n",
      "677      0      0               4138    9.8417      NaN        S  \n",
      "570      0      0        S.W./PP 752   10.5000      NaN        S  \n",
      "744      0      0  STON/O 2. 3101288    7.9250      NaN        S  \n",
      "109      1      0             371110   24.1500      NaN        Q  \n",
      "97       0      1           PC 17759   63.3583  D10 D12        C  \n",
      "25       1      5             347077   31.3875      NaN        S  \n"
     ]
    }
   ],
   "source": [
    "# final test\n",
    "best_clf = pipe_filtered_2\n",
    "best_clf.fit(X,y)\n",
    "pred = best_clf.predict(X_test)\n",
    "print('score: ',roc_auc_score(y_test,pred))\n",
    "print(confusion_matrix(y_test,pred))\n",
    "print('fp: \\n', X_test[(y_test != pred) & (y_test == 0)])\n",
    "fn = X_test[(y_test != pred) & (y_test == 1)]\n",
    "print('\\n\\nfn: \\n', fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pclass        int64\n",
       "Name         object\n",
       "Sex          object\n",
       "Age         float64\n",
       "SibSp         int64\n",
       "Parch         int64\n",
       "Ticket       object\n",
       "Fare        float64\n",
       "Cabin        object\n",
       "Embarked     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.dtypes\n",
    "# X.iloc[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Pclass                  Name   Sex   Age  SibSp  Parch Ticket  Fare Cabin  \\\n",
      "0       1  Shumacher, Mr. Ariel  male  21.0      2      4   None  30.0    E3   \n",
      "\n",
      "  Embarked  \n",
      "0        S   \n",
      " Pclass        int64\n",
      "Name         object\n",
      "Sex          object\n",
      "Age         float64\n",
      "SibSp         int64\n",
      "Parch         int64\n",
      "Ticket       object\n",
      "Fare        float64\n",
      "Cabin        object\n",
      "Embarked     object\n",
      "dtype: object\n",
      "will you survive?:  [ True] \n",
      "with probability: [[0.38564473 0.61435527]]\n"
     ]
    }
   ],
   "source": [
    "newbie = [1,'Shumacher, Mr. Ariel','male',21,2,4,None,30,'E3','S']\n",
    "def get_df_like_titanic(x):\n",
    "    res = X.iloc[0:0,:].copy()\n",
    "    res.append(x)\n",
    "    return res\n",
    "def will_you_survive(*params):\n",
    "    res = X.iloc[0:0,:].copy()\n",
    "    df = pd.DataFrame(np.atleast_2d(params),columns=X.columns)\n",
    "    for x in X.columns:\n",
    "        df[x]=df[x].astype(X[x].dtypes.name)\n",
    "    print(df,'\\n',df.dtypes)\n",
    "    print('will you survive?: ',best_clf.predict(df)==1,\n",
    "          '\\nwith probability:',best_clf.predict_proba(df))\n",
    "will_you_survive(*newbie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyCharm (Project)",
   "language": "python",
   "name": "pycharm-1a86bc75"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}