Random Forest Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fraud Detection Zindi Project\n",
    "\n",
    "By Atwine Mugume\n",
    "\n",
    "Some of the resources used here include these articles that educate on imbalanced data\n",
    "\n",
    "- [Dealing with Imbalanced Data](https://towardsdatascience.com/methods-for-dealing-with-imbalanced-data-5b761be45a18)\n",
    "- [Handling Imbalanced Datasets: Predicting Credit Card Fraud](https://medium.com/coinmonks/handling-imbalanced-datasets-predicting-credit-card-fraud-544f5e74e0fd)\n",
    "- [Tackling Class Imbalance](https://www.kaggle.com/shahules/tackling-class-imbalance)  thanks to Shahules786's kernel\n",
    "- [Great article on scaling data](https://www.kaggle.com/discdiver/guide-to-scaling-and-standardizing?source=post_page---------------------------)\n",
    "-[performance-metrics-for-classification-problems](https://medium.com/thalus-ai/performance-metrics-for-classification-problems-in-machine-learning-part-i-b085d432082b)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['training.csv', 'test.csv', '.ipynb_checkpoints', 'Xente_Variable_Definitions.csv', 'fastai-xente-project-v.ipynb', 'Random Forest Classifier.ipynb']\n",
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 6.44 µs\n",
      "Importing fastai libraries\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"./\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "%time\n",
    "print('Importing fastai libraries')\n",
    "from fastai.imports import *\n",
    "from fastai.structured import *\n",
    "\n",
    "from pandas_summary import DataFrameSummary\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from IPython.display import display\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Readind the data!\n"
     ]
    }
   ],
   "source": [
    "#let's take a quick look at the data\n",
    "#since we have a time variable we have to parse the dates\n",
    "train = pd.read_csv('./training.csv', low_memory=True, \n",
    "                     parse_dates=[\"TransactionStartTime\"])\n",
    "test = pd.read_csv('./test.csv',low_memory=True, \n",
    "                     parse_dates=[\"TransactionStartTime\"])\n",
    "\n",
    "print('Done Readind the data!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column Name</th>\n",
       "      <th>Definition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId</td>\n",
       "      <td>Unique �transaction identifier on platform</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId</td>\n",
       "      <td>Unique number assigned to a batch of transacti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AccountId</td>\n",
       "      <td>Unique number identifying the customer on plat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SubscriptionId</td>\n",
       "      <td>Unique number identifying the customer subscri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CustomerId</td>\n",
       "      <td>Unique identifier attached to Account</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CurrencyCode</td>\n",
       "      <td>Country currency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CountryCode</td>\n",
       "      <td>Numerical geographical code of country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ProviderId</td>\n",
       "      <td>Source provider of Item �bought.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ProductId</td>\n",
       "      <td>Item name being bought.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ProductCategory</td>\n",
       "      <td>ProductIds are organized into these broader pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ChannelId</td>\n",
       "      <td>Identifies if customer used web,Android, IOS, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Amount</td>\n",
       "      <td>Value of the transaction. Positive for debits ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Value</td>\n",
       "      <td>Absolute value of the amount</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TransactionStartTime</td>\n",
       "      <td>Transaction start time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PricingStrategy</td>\n",
       "      <td>Category of Xente's pricing structure for merc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FraudResult</td>\n",
       "      <td>Fraud status of transaction 1 -yes or 0-No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Column Name                                         Definition\n",
       "0          TransactionId         Unique �transaction identifier on platform\n",
       "1                BatchId  Unique number assigned to a batch of transacti...\n",
       "2              AccountId  Unique number identifying the customer on plat...\n",
       "3         SubscriptionId  Unique number identifying the customer subscri...\n",
       "4             CustomerId              Unique identifier attached to Account\n",
       "5           CurrencyCode                                   Country currency\n",
       "6            CountryCode             Numerical geographical code of country\n",
       "7             ProviderId                   Source provider of Item �bought.\n",
       "8              ProductId                            Item name being bought.\n",
       "9        ProductCategory  ProductIds are organized into these broader pr...\n",
       "10             ChannelId  Identifies if customer used web,Android, IOS, ...\n",
       "11                Amount  Value of the transaction. Positive for debits ...\n",
       "12                 Value                       Absolute value of the amount\n",
       "13  TransactionStartTime                             Transaction start time\n",
       "14       PricingStrategy  Category of Xente's pricing structure for merc...\n",
       "15           FraudResult         Fraud status of transaction 1 -yes or 0-No"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a description of the variables here to keep refering to them\n",
    "feature_desc = pd.read_csv('./Xente_Variable_Definitions.csv')\n",
    "feature_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 16), (45019, 15))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#what does the data look like\n",
    "train.shape,test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are datasets which we have to look at carefully because they have a time variable. We have to make sure the time is progressive.\n",
    "That is we are having the first rows from a time before those at the end of the dataset\n",
    "\n",
    "For example if the first row is on the date 9/Jan/2019 the last row should not be the same date...it should be maybe 25/Oct/2021 so we can have a progression. \n",
    "\n",
    "Therefore our model will be used to tell the future."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId_76871</td>\n",
       "      <td>BatchId_36123</td>\n",
       "      <td>AccountId_3957</td>\n",
       "      <td>SubscriptionId_887</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TransactionId_73770</td>\n",
       "      <td>BatchId_15642</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TransactionId_26203</td>\n",
       "      <td>BatchId_53941</td>\n",
       "      <td>AccountId_4229</td>\n",
       "      <td>SubscriptionId_222</td>\n",
       "      <td>CustomerId_4683</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_1</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TransactionId_380</td>\n",
       "      <td>BatchId_102363</td>\n",
       "      <td>AccountId_648</td>\n",
       "      <td>SubscriptionId_2185</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_1</td>\n",
       "      <td>ProductId_21</td>\n",
       "      <td>utility_bill</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TransactionId_28195</td>\n",
       "      <td>BatchId_38780</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         TransactionId         BatchId       AccountId       SubscriptionId  \\\n",
       "0  TransactionId_76871   BatchId_36123  AccountId_3957   SubscriptionId_887   \n",
       "1  TransactionId_73770   BatchId_15642  AccountId_4841  SubscriptionId_3829   \n",
       "2  TransactionId_26203   BatchId_53941  AccountId_4229   SubscriptionId_222   \n",
       "3    TransactionId_380  BatchId_102363   AccountId_648  SubscriptionId_2185   \n",
       "4  TransactionId_28195   BatchId_38780  AccountId_4841  SubscriptionId_3829   \n",
       "\n",
       "        CustomerId CurrencyCode  CountryCode    ProviderId     ProductId  \\\n",
       "0  CustomerId_4406          UGX          256  ProviderId_6  ProductId_10   \n",
       "1  CustomerId_4406          UGX          256  ProviderId_4   ProductId_6   \n",
       "2  CustomerId_4683          UGX          256  ProviderId_6   ProductId_1   \n",
       "3   CustomerId_988          UGX          256  ProviderId_1  ProductId_21   \n",
       "4   CustomerId_988          UGX          256  ProviderId_4   ProductId_6   \n",
       "\n",
       "      ProductCategory    ChannelId   Amount  Value      TransactionStartTime  \\\n",
       "0             airtime  ChannelId_3   1000.0   1000 2018-11-15 02:18:49+00:00   \n",
       "1  financial_services  ChannelId_2    -20.0     20 2018-11-15 02:19:08+00:00   \n",
       "2             airtime  ChannelId_3    500.0    500 2018-11-15 02:44:21+00:00   \n",
       "3        utility_bill  ChannelId_3  20000.0  21800 2018-11-15 03:32:55+00:00   \n",
       "4  financial_services  ChannelId_2   -644.0    644 2018-11-15 03:34:21+00:00   \n",
       "\n",
       "   PricingStrategy  FraudResult  \n",
       "0                2            0  \n",
       "1                2            0  \n",
       "2                2            0  \n",
       "3                2            0  \n",
       "4                2            0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a snippet of the head of the two datasets\n",
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95657</th>\n",
       "      <td>TransactionId_89881</td>\n",
       "      <td>BatchId_96668</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3078</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 09:54:09+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95658</th>\n",
       "      <td>TransactionId_91597</td>\n",
       "      <td>BatchId_3503</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 09:54:25+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95659</th>\n",
       "      <td>TransactionId_82501</td>\n",
       "      <td>BatchId_118602</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2019-02-13 09:54:35+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95660</th>\n",
       "      <td>TransactionId_136354</td>\n",
       "      <td>BatchId_70924</td>\n",
       "      <td>AccountId_1346</td>\n",
       "      <td>SubscriptionId_652</td>\n",
       "      <td>CustomerId_1709</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_19</td>\n",
       "      <td>tv</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:01:10+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95661</th>\n",
       "      <td>TransactionId_35670</td>\n",
       "      <td>BatchId_29317</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_1709</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-60.0</td>\n",
       "      <td>60</td>\n",
       "      <td>2019-02-13 10:01:28+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              TransactionId         BatchId       AccountId  \\\n",
       "95657   TransactionId_89881   BatchId_96668  AccountId_4841   \n",
       "95658   TransactionId_91597    BatchId_3503  AccountId_3439   \n",
       "95659   TransactionId_82501  BatchId_118602  AccountId_4841   \n",
       "95660  TransactionId_136354   BatchId_70924  AccountId_1346   \n",
       "95661   TransactionId_35670   BatchId_29317  AccountId_4841   \n",
       "\n",
       "            SubscriptionId       CustomerId CurrencyCode  CountryCode  \\\n",
       "95657  SubscriptionId_3829  CustomerId_3078          UGX          256   \n",
       "95658  SubscriptionId_2643  CustomerId_3874          UGX          256   \n",
       "95659  SubscriptionId_3829  CustomerId_3874          UGX          256   \n",
       "95660   SubscriptionId_652  CustomerId_1709          UGX          256   \n",
       "95661  SubscriptionId_3829  CustomerId_1709          UGX          256   \n",
       "\n",
       "         ProviderId     ProductId     ProductCategory    ChannelId  Amount  \\\n",
       "95657  ProviderId_4   ProductId_6  financial_services  ChannelId_2 -1000.0   \n",
       "95658  ProviderId_6  ProductId_10             airtime  ChannelId_3  1000.0   \n",
       "95659  ProviderId_4   ProductId_6  financial_services  ChannelId_2   -20.0   \n",
       "95660  ProviderId_6  ProductId_19                  tv  ChannelId_3  3000.0   \n",
       "95661  ProviderId_4   ProductId_6  financial_services  ChannelId_2   -60.0   \n",
       "\n",
       "       Value      TransactionStartTime  PricingStrategy  FraudResult  \n",
       "95657   1000 2019-02-13 09:54:09+00:00                2            0  \n",
       "95658   1000 2019-02-13 09:54:25+00:00                2            0  \n",
       "95659     20 2019-02-13 09:54:35+00:00                2            0  \n",
       "95660   3000 2019-02-13 10:01:10+00:00                2            0  \n",
       "95661     60 2019-02-13 10:01:28+00:00                2            0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId_50600</td>\n",
       "      <td>BatchId_35028</td>\n",
       "      <td>AccountId_2441</td>\n",
       "      <td>SubscriptionId_4426</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TransactionId_95109</td>\n",
       "      <td>BatchId_45139</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_15</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TransactionId_47357</td>\n",
       "      <td>BatchId_74887</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-50.0</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TransactionId_28185</td>\n",
       "      <td>BatchId_11025</td>\n",
       "      <td>AccountId_2685</td>\n",
       "      <td>SubscriptionId_4626</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:02:38+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TransactionId_22140</td>\n",
       "      <td>BatchId_29804</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-60.0</td>\n",
       "      <td>60</td>\n",
       "      <td>2019-02-13 10:02:58+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         TransactionId        BatchId       AccountId       SubscriptionId  \\\n",
       "0  TransactionId_50600  BatchId_35028  AccountId_2441  SubscriptionId_4426   \n",
       "1  TransactionId_95109  BatchId_45139  AccountId_3439  SubscriptionId_2643   \n",
       "2  TransactionId_47357  BatchId_74887  AccountId_4841  SubscriptionId_3829   \n",
       "3  TransactionId_28185  BatchId_11025  AccountId_2685  SubscriptionId_4626   \n",
       "4  TransactionId_22140  BatchId_29804  AccountId_4841  SubscriptionId_3829   \n",
       "\n",
       "        CustomerId CurrencyCode  CountryCode    ProviderId     ProductId  \\\n",
       "0  CustomerId_2857          UGX          256  ProviderId_5   ProductId_3   \n",
       "1  CustomerId_3874          UGX          256  ProviderId_5  ProductId_15   \n",
       "2  CustomerId_2857          UGX          256  ProviderId_4   ProductId_6   \n",
       "3  CustomerId_3105          UGX          256  ProviderId_5  ProductId_10   \n",
       "4  CustomerId_3105          UGX          256  ProviderId_4   ProductId_6   \n",
       "\n",
       "      ProductCategory    ChannelId  Amount  Value      TransactionStartTime  \\\n",
       "0             airtime  ChannelId_3  1000.0   1000 2019-02-13 10:01:40+00:00   \n",
       "1  financial_services  ChannelId_3  2000.0   2000 2019-02-13 10:02:12+00:00   \n",
       "2  financial_services  ChannelId_2   -50.0     50 2019-02-13 10:02:30+00:00   \n",
       "3             airtime  ChannelId_3  3000.0   3000 2019-02-13 10:02:38+00:00   \n",
       "4  financial_services  ChannelId_2   -60.0     60 2019-02-13 10:02:58+00:00   \n",
       "\n",
       "   PricingStrategy  \n",
       "0                4  \n",
       "1                2  \n",
       "2                2  \n",
       "3                4  \n",
       "4                2  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's get snippet of the test data\n",
    "test.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>45014</th>\n",
       "      <td>TransactionId_116198</td>\n",
       "      <td>BatchId_117348</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_5225</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>-5000.0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2019-03-14 23:56:28+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45015</th>\n",
       "      <td>TransactionId_24829</td>\n",
       "      <td>BatchId_53630</td>\n",
       "      <td>AccountId_4803</td>\n",
       "      <td>SubscriptionId_1738</td>\n",
       "      <td>CustomerId_5272</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_3</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>2019-03-14 23:57:38+00:00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45016</th>\n",
       "      <td>TransactionId_102920</td>\n",
       "      <td>BatchId_112209</td>\n",
       "      <td>AccountId_4817</td>\n",
       "      <td>SubscriptionId_4613</td>\n",
       "      <td>CustomerId_5286</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_3</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>2019-03-14 23:59:17+00:00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45017</th>\n",
       "      <td>TransactionId_128439</td>\n",
       "      <td>BatchId_58734</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_5286</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>-5000.0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2019-03-14 23:59:39+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45018</th>\n",
       "      <td>TransactionId_45927</td>\n",
       "      <td>BatchId_98308</td>\n",
       "      <td>AccountId_4593</td>\n",
       "      <td>SubscriptionId_3866</td>\n",
       "      <td>CustomerId_5057</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>3500</td>\n",
       "      <td>2019-03-14 23:59:51+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              TransactionId         BatchId       AccountId  \\\n",
       "45014  TransactionId_116198  BatchId_117348  AccountId_4841   \n",
       "45015   TransactionId_24829   BatchId_53630  AccountId_4803   \n",
       "45016  TransactionId_102920  BatchId_112209  AccountId_4817   \n",
       "45017  TransactionId_128439   BatchId_58734  AccountId_4841   \n",
       "45018   TransactionId_45927   BatchId_98308  AccountId_4593   \n",
       "\n",
       "            SubscriptionId       CustomerId CurrencyCode  CountryCode  \\\n",
       "45014  SubscriptionId_3829  CustomerId_5225          UGX          256   \n",
       "45015  SubscriptionId_1738  CustomerId_5272          UGX          256   \n",
       "45016  SubscriptionId_4613  CustomerId_5286          UGX          256   \n",
       "45017  SubscriptionId_3829  CustomerId_5286          UGX          256   \n",
       "45018  SubscriptionId_3866  CustomerId_5057          UGX          256   \n",
       "\n",
       "         ProviderId     ProductId     ProductCategory    ChannelId   Amount  \\\n",
       "45014  ProviderId_4   ProductId_6  financial_services  ChannelId_3  -5000.0   \n",
       "45015  ProviderId_3  ProductId_10             airtime  ChannelId_3  10000.0   \n",
       "45016  ProviderId_3  ProductId_10             airtime  ChannelId_3  10000.0   \n",
       "45017  ProviderId_4   ProductId_6  financial_services  ChannelId_3  -5000.0   \n",
       "45018  ProviderId_5   ProductId_3             airtime  ChannelId_3   3500.0   \n",
       "\n",
       "       Value      TransactionStartTime  PricingStrategy  \n",
       "45014   5000 2019-03-14 23:56:28+00:00                2  \n",
       "45015  10000 2019-03-14 23:57:38+00:00                1  \n",
       "45016  10000 2019-03-14 23:59:17+00:00                1  \n",
       "45017   5000 2019-03-14 23:59:39+00:00                2  \n",
       "45018   3500 2019-03-14 23:59:51+00:00                4  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 95662 entries, 0 to 95661\n",
      "Data columns (total 16 columns):\n",
      "TransactionId           95662 non-null object\n",
      "BatchId                 95662 non-null object\n",
      "AccountId               95662 non-null object\n",
      "SubscriptionId          95662 non-null object\n",
      "CustomerId              95662 non-null object\n",
      "CurrencyCode            95662 non-null object\n",
      "CountryCode             95662 non-null int64\n",
      "ProviderId              95662 non-null object\n",
      "ProductId               95662 non-null object\n",
      "ProductCategory         95662 non-null object\n",
      "ChannelId               95662 non-null object\n",
      "Amount                  95662 non-null float64\n",
      "Value                   95662 non-null int64\n",
      "TransactionStartTime    95662 non-null datetime64[ns, UTC]\n",
      "PricingStrategy         95662 non-null int64\n",
      "FraudResult             95662 non-null int64\n",
      "dtypes: datetime64[ns, UTC](1), float64(1), int64(4), object(10)\n",
      "memory usage: 11.7+ MB\n"
     ]
    }
   ],
   "source": [
    "#Let's take a peek at the data into\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 16), (45019, 15))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's look at the shapes of the data\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Data Analysis\n",
    "\n",
    "### Encoding\n",
    "\n",
    "I have only discovered that the way you encode the data you have can affect how well your model will function. I have found some great resources here in these [article](https://medium.com/data-design/visiting-categorical-features-and-encoding-in-decision-trees-53400fa65931)\n",
    "\n",
    "-This article is also a [good one](https://towardsdatascience.com/smarter-ways-to-encode-categorical-data-for-machine-learning-part-1-of-3-6dca2f71b159)\n",
    "\n",
    "What stands out the most is these two points:\n",
    "\n",
    "- Categorical features with large cardinalities/features (over 1000): `Binary Encoding` is ideal\n",
    "- Categorical features with small cardinalities/features (less than 1000): `Numeric Encoding` is ideal\n",
    "\n",
    "As an esample here is a code below from the library: binaryencoder\n",
    "\n",
    "```python \n",
    "from category_encoders import *\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "# prepare some data\n",
    "bunch = load_boston()\n",
    "y = bunch.target\n",
    "X = pd.DataFrame(bunch.data, columns=bunch.feature_names)\n",
    "\n",
    "# use binary encoding to encode two categorical features\n",
    "enc = BinaryEncoder(cols=['CHAS', 'RAD']).fit(X)\n",
    "\n",
    "# transform the dataset\n",
    "numeric_dataset = enc.transform(X)\n",
    "\n",
    "\n",
    "## A supervised example\n",
    "\n",
    "from category_encoders import *\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "# prepare some data\n",
    "bunch = load_boston()\n",
    "y_train = bunch.target[0:250]\n",
    "y_test = bunch.target[250:506]\n",
    "X_train = pd.DataFrame(bunch.data[0:250], columns=bunch.feature_names)\n",
    "X_test = pd.DataFrame(bunch.data[250:506], columns=bunch.feature_names)\n",
    "\n",
    "# use target encoding to encode two categorical features\n",
    "enc = TargetEncoder(cols=['CHAS', 'RAD']).fit(X_train, y_train)\n",
    "\n",
    "# transform the datasets\n",
    "training_numeric_dataset = enc.transform(X_train, y_train)\n",
    "testing_numeric_dataset = enc.transform(X_test)\n",
    "\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TransactionId has 95662\n",
      "BatchId has 94809\n",
      "AccountId has 3633\n",
      "SubscriptionId has 3627\n",
      "CustomerId has 3742\n",
      "CurrencyCode has 1\n",
      "CountryCode has 1\n",
      "ProviderId has 6\n",
      "ProductId has 23\n",
      "ProductCategory has 9\n",
      "ChannelId has 4\n",
      "Amount has 1676\n",
      "Value has 1517\n",
      "TransactionStartTime has 94556\n",
      "PricingStrategy has 4\n",
      "FraudResult has 2\n"
     ]
    }
   ],
   "source": [
    "#how unique is the batchId?\n",
    "#i need to know if I should leave it in the conversation\n",
    "for col in train.columns:\n",
    "    print(col,'has', train[col].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's install the package with the encoders that make sense for this kind of data with only 10+ features\n",
    "#%pip install category_encoders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lil Function\n",
    "\n",
    "One fo the things I have learnt is to make code reusable, so I am going to write a function that will help us encode without having to repeat the code so much.\n",
    "\n",
    "`Why is this function so important?`\n",
    "\n",
    "I have been having memory issues wen I use the pd.getdummies. \n",
    "I normally have memory limited to 13Gb and as a result this method of replacing the categorical values with simple codes keeps the memory usage under 200Mbs. Which is quite phenomenal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#little function to help us encode data\n",
    "def lil_encoder(df,col):\n",
    "    df[col] = train[col].astype('category')\n",
    "    \n",
    "    #let's create a new column of the categorical features\n",
    "    #df[col] = df[col].cat.codes\n",
    "    \n",
    "    return df.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we need to keep the ids of the test data\n",
    "fin_Id = test['TransactionId']\n",
    "y = train['FraudResult']\n",
    "\n",
    "#let's now drop them from their original datasets\n",
    "test = test.drop(columns=['TransactionId'])\n",
    "train = train.drop(columns=['FraudResult','TransactionId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 14), (45019, 14))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#i want to see if the datasets are aligned, when they are not aligned one gets problems in fitting the data\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BatchId_35028</td>\n",
       "      <td>AccountId_2441</td>\n",
       "      <td>SubscriptionId_4426</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId_45139</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_15</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BatchId_74887</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-50.0</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BatchId_11025</td>\n",
       "      <td>AccountId_2685</td>\n",
       "      <td>SubscriptionId_4626</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:02:38+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         BatchId       AccountId       SubscriptionId       CustomerId  \\\n",
       "0  BatchId_35028  AccountId_2441  SubscriptionId_4426  CustomerId_2857   \n",
       "1  BatchId_45139  AccountId_3439  SubscriptionId_2643  CustomerId_3874   \n",
       "2  BatchId_74887  AccountId_4841  SubscriptionId_3829  CustomerId_2857   \n",
       "3  BatchId_11025  AccountId_2685  SubscriptionId_4626  CustomerId_3105   \n",
       "\n",
       "  CurrencyCode  CountryCode    ProviderId     ProductId     ProductCategory  \\\n",
       "0          UGX          256  ProviderId_5   ProductId_3             airtime   \n",
       "1          UGX          256  ProviderId_5  ProductId_15  financial_services   \n",
       "2          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "3          UGX          256  ProviderId_5  ProductId_10             airtime   \n",
       "\n",
       "     ChannelId  Amount  Value      TransactionStartTime  PricingStrategy  \n",
       "0  ChannelId_3  1000.0   1000 2019-02-13 10:01:40+00:00                4  \n",
       "1  ChannelId_3  2000.0   2000 2019-02-13 10:02:12+00:00                2  \n",
       "2  ChannelId_2   -50.0     50 2019-02-13 10:02:30+00:00                2  \n",
       "3  ChannelId_3  3000.0   3000 2019-02-13 10:02:38+00:00                4  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BatchId_36123</td>\n",
       "      <td>AccountId_3957</td>\n",
       "      <td>SubscriptionId_887</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId_15642</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BatchId_53941</td>\n",
       "      <td>AccountId_4229</td>\n",
       "      <td>SubscriptionId_222</td>\n",
       "      <td>CustomerId_4683</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_1</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BatchId_102363</td>\n",
       "      <td>AccountId_648</td>\n",
       "      <td>SubscriptionId_2185</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_1</td>\n",
       "      <td>ProductId_21</td>\n",
       "      <td>utility_bill</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BatchId_38780</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          BatchId       AccountId       SubscriptionId       CustomerId  \\\n",
       "0   BatchId_36123  AccountId_3957   SubscriptionId_887  CustomerId_4406   \n",
       "1   BatchId_15642  AccountId_4841  SubscriptionId_3829  CustomerId_4406   \n",
       "2   BatchId_53941  AccountId_4229   SubscriptionId_222  CustomerId_4683   \n",
       "3  BatchId_102363   AccountId_648  SubscriptionId_2185   CustomerId_988   \n",
       "4   BatchId_38780  AccountId_4841  SubscriptionId_3829   CustomerId_988   \n",
       "\n",
       "  CurrencyCode  CountryCode    ProviderId     ProductId     ProductCategory  \\\n",
       "0          UGX          256  ProviderId_6  ProductId_10             airtime   \n",
       "1          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "2          UGX          256  ProviderId_6   ProductId_1             airtime   \n",
       "3          UGX          256  ProviderId_1  ProductId_21        utility_bill   \n",
       "4          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "\n",
       "     ChannelId   Amount  Value      TransactionStartTime  PricingStrategy  \\\n",
       "0  ChannelId_3   1000.0   1000 2018-11-15 02:18:49+00:00                2   \n",
       "1  ChannelId_2    -20.0     20 2018-11-15 02:19:08+00:00                2   \n",
       "2  ChannelId_3    500.0    500 2018-11-15 02:44:21+00:00                2   \n",
       "3  ChannelId_3  20000.0  21800 2018-11-15 03:32:55+00:00                2   \n",
       "4  ChannelId_2   -644.0    644 2018-11-15 03:34:21+00:00                2   \n",
       "\n",
       "  Identifier  \n",
       "0      Train  \n",
       "1      Train  \n",
       "2      Train  \n",
       "3      Train  \n",
       "4      Train  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let me put the datasets together so that I can work on them at the same time\n",
    "train['Identifier']='Train'\n",
    "test['Identifier']='Test'\n",
    "\n",
    "ls = [train, test]\n",
    "data = pd.concat(ls)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's encode all the data\n",
    "ls_ = ['AccountId','CustomerId','ProviderId','ProductId','ProductCategory','ChannelId','BatchId','SubscriptionId']\n",
    "\n",
    "#this is the for loop that calls the function and makes the lil_function work\n",
    "for col in ls_:\n",
    "    lil_encoder(data,col)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have all the columns in the list above transformed from object to categorical, now we are going to encode them with a categorical encoder. I have chosen to update my encoding method because of some of the articles I have read on this topic, they insist that we shd use Binary encoders for small datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from category_encoders import *\n",
    "\n",
    "# use binary encoding to encode two categorical features\n",
    "enc = BinaryEncoder(cols=['AccountId','CustomerId','ProviderId','ProductId','ProductCategory','ChannelId','BatchId','SubscriptionId']).fit(data)\n",
    "\n",
    "data = enc.transform(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>ProductCategory_3</th>\n",
       "      <th>ProductCategory_4</th>\n",
       "      <th>ChannelId_0</th>\n",
       "      <th>ChannelId_1</th>\n",
       "      <th>ChannelId_2</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "4          0          0          1          0          1          1   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  ProductCategory_3  \\\n",
       "0          0          1          1          1  ...                  0   \n",
       "1          1          1          0          0  ...                  1   \n",
       "2          1          0          1          1  ...                  0   \n",
       "3          0          1          1          1  ...                  0   \n",
       "4          1          1          1          1  ...                  1   \n",
       "\n",
       "   ProductCategory_4  ChannelId_0  ChannelId_1  ChannelId_2   Amount  Value  \\\n",
       "0                  1            0            1            1   1000.0   1000   \n",
       "1                  1            0            1            0    -20.0     20   \n",
       "2                  1            0            1            1    500.0    500   \n",
       "3                  1            0            1            1  20000.0  21800   \n",
       "4                  1            0            1            0   -644.0    644   \n",
       "\n",
       "       TransactionStartTime  PricingStrategy  Identifier  \n",
       "0 2018-11-15 02:18:49+00:00                2       Train  \n",
       "1 2018-11-15 02:19:08+00:00                2       Train  \n",
       "2 2018-11-15 02:44:21+00:00                2       Train  \n",
       "3 2018-11-15 03:32:55+00:00                2       Train  \n",
       "4 2018-11-15 03:34:21+00:00                2       Train  \n",
       "\n",
       "[5 rows x 82 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 79), (45019, 79))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dropping some features: there are some features we can see that will not contribute to the ML process\n",
    "\n",
    "#let's split the data\n",
    "train = data[data['Identifier']=='Train']\n",
    "test = data[data['Identifier']=='Test']\n",
    "\n",
    "#now let's drop the useless columns from the datasets\n",
    "train = train.drop(columns=['CurrencyCode','CountryCode','Amount'])\n",
    "test = test.drop(columns=['CurrencyCode','CountryCode','Amount'])\n",
    "\n",
    "#let's see the new shapes\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>ProductCategory_2</th>\n",
       "      <th>ProductCategory_3</th>\n",
       "      <th>ProductCategory_4</th>\n",
       "      <th>ChannelId_0</th>\n",
       "      <th>ChannelId_1</th>\n",
       "      <th>ChannelId_2</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  ProductCategory_2  \\\n",
       "0          0          1          1          1  ...                  0   \n",
       "1          1          1          0          0  ...                  0   \n",
       "2          1          0          1          1  ...                  0   \n",
       "\n",
       "   ProductCategory_3  ProductCategory_4  ChannelId_0  ChannelId_1  \\\n",
       "0                  0                  1            0            1   \n",
       "1                  1                  1            0            1   \n",
       "2                  0                  1            0            1   \n",
       "\n",
       "   ChannelId_2  Value      TransactionStartTime  PricingStrategy  Identifier  \n",
       "0            1   1000 2019-02-13 10:01:40+00:00                4        Test  \n",
       "1            0   2000 2019-02-13 10:02:12+00:00                2        Test  \n",
       "2            1     50 2019-02-13 10:02:30+00:00                2        Test  \n",
       "\n",
       "[3 rows x 79 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have tried to load the date time function from fastai but it has failed so i am going to copy it manually and use it here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datepart(df, fldname, drop=True, time=False):\n",
    "    \"\"\"add_datepart converts a column of df from a datetime64 to many columns containing\n",
    "    the information from the date. This applies changes inplace.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas data frame. df gain several new columns.\n",
    "    fldname: A string that is the name of the date column you wish to expand.\n",
    "        If it is not a datetime64 series, it will be converted to one with pd.to_datetime.\n",
    "    drop: If true then the original date column will be removed.\n",
    "    time: If true time features: Hour, Minute, Second will be added.\n",
    "\n",
    "    Examples:\n",
    "    ---------\n",
    "\n",
    "    >>> df = pd.DataFrame({ 'A' : pd.to_datetime(['3/11/2000', '3/12/2000', '3/13/2000'], infer_datetime_format=False) })\n",
    "    >>> df\n",
    "\n",
    "        A\n",
    "    0   2000-03-11\n",
    "    1   2000-03-12\n",
    "    2   2000-03-13\n",
    "\n",
    "    >>> add_datepart(df, 'A')\n",
    "    >>> df\n",
    "\n",
    "        AYear AMonth AWeek ADay ADayofweek ADayofyear AIs_month_end AIs_month_start AIs_quarter_end AIs_quarter_start AIs_year_end AIs_year_start AElapsed\n",
    "    0   2000  3      10    11   5          71         False         False           False           False             False        False          952732800\n",
    "    1   2000  3      10    12   6          72         False         False           False           False             False        False          952819200\n",
    "    2   2000  3      11    13   0          73         False         False           False           False             False        False          952905600\n",
    "    \"\"\"\n",
    "    fld = df[fldname]\n",
    "    fld_dtype = fld.dtype\n",
    "    if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):\n",
    "        fld_dtype = np.datetime64\n",
    "\n",
    "    if not np.issubdtype(fld_dtype, np.datetime64):\n",
    "        df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)\n",
    "    targ_pre = re.sub('[Dd]ate$', '', fldname)\n",
    "    attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',\n",
    "            'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']\n",
    "    if time: attr = attr + ['Hour', 'Minute', 'Second']\n",
    "    for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())\n",
    "    df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9\n",
    "    if drop: df.drop(fldname, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's add the datetime variables\n",
    "#this is a function in the fastai lib that deals with all variables of time, ie it converts the datatime column into many others\n",
    "add_datepart(test,'TransactionStartTime')\n",
    "add_datepart(train,'TransactionStartTime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDay</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542249861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542252775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  TransactionStartTimeDay  \\\n",
       "0          0          1          1          1  ...                       15   \n",
       "1          1          1          0          0  ...                       15   \n",
       "2          1          0          1          1  ...                       15   \n",
       "3          0          1          1          1  ...                       15   \n",
       "\n",
       "   TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "0                              3                            319   \n",
       "1                              3                            319   \n",
       "2                              3                            319   \n",
       "3                              3                            319   \n",
       "\n",
       "   TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "0                             False                               False   \n",
       "1                             False                               False   \n",
       "2                             False                               False   \n",
       "3                             False                               False   \n",
       "\n",
       "   TransactionStartTimeIs_quarter_end  TransactionStartTimeIs_quarter_start  \\\n",
       "0                               False                                 False   \n",
       "1                               False                                 False   \n",
       "2                               False                                 False   \n",
       "3                               False                                 False   \n",
       "\n",
       "   TransactionStartTimeIs_year_end  TransactionStartTimeIs_year_start  \\\n",
       "0                            False                              False   \n",
       "1                            False                              False   \n",
       "2                            False                              False   \n",
       "3                            False                              False   \n",
       "\n",
       "   TransactionStartTimeElapsed  \n",
       "0                   1542248329  \n",
       "1                   1542248348  \n",
       "2                   1542249861  \n",
       "3                   1542252775  \n",
       "\n",
       "[4 rows x 91 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a look at the dataframe now and see how its looks like after we add the datetime variables\n",
    "train.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's remove the identifier columns so that we have only the features we need to use otherwise we will have problems.\n",
    "train = train.drop(columns='Identifier')\n",
    "test = test.drop(columns='Identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDay</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542249861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542252775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  TransactionStartTimeDay  \\\n",
       "0          0          1          1          1  ...                       15   \n",
       "1          1          1          0          0  ...                       15   \n",
       "2          1          0          1          1  ...                       15   \n",
       "3          0          1          1          1  ...                       15   \n",
       "\n",
       "   TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "0                              3                            319   \n",
       "1                              3                            319   \n",
       "2                              3                            319   \n",
       "3                              3                            319   \n",
       "\n",
       "   TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "0                             False                               False   \n",
       "1                             False                               False   \n",
       "2                             False                               False   \n",
       "3                             False                               False   \n",
       "\n",
       "   TransactionStartTimeIs_quarter_end  TransactionStartTimeIs_quarter_start  \\\n",
       "0                               False                                 False   \n",
       "1                               False                                 False   \n",
       "2                               False                                 False   \n",
       "3                               False                                 False   \n",
       "\n",
       "   TransactionStartTimeIs_year_end  TransactionStartTimeIs_year_start  \\\n",
       "0                            False                              False   \n",
       "1                            False                              False   \n",
       "2                            False                              False   \n",
       "3                            False                              False   \n",
       "\n",
       "   TransactionStartTimeElapsed  \n",
       "0                   1542248329  \n",
       "1                   1542248348  \n",
       "2                   1542249861  \n",
       "3                   1542252775  \n",
       "\n",
       "[4 rows x 90 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 90), (45019, 90))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's see the shapes of the data now\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Positive Correlations:\n",
      " SubscriptionId_2     0.033930\n",
      "ChannelId_2          0.034272\n",
      "ProductId_3          0.035686\n",
      "ProductId_5          0.045169\n",
      "ProductId_4          0.049997\n",
      "ProviderId_3         0.074962\n",
      "Value                0.566739\n",
      "FraudResult          1.000000\n",
      "BatchId_0                 NaN\n",
      "AccountId_0               NaN\n",
      "SubscriptionId_0          NaN\n",
      "CustomerId_0              NaN\n",
      "ProviderId_0              NaN\n",
      "ProductId_0               NaN\n",
      "ProductCategory_0         NaN\n",
      "Name: FraudResult, dtype: float64\n",
      "\n",
      "Most Negative Correlations:\n",
      " ProviderId_1                    -0.083548\n",
      "ProductId_1                     -0.049321\n",
      "PricingStrategy                 -0.033821\n",
      "SubscriptionId_1                -0.029144\n",
      "SubscriptionId_4                -0.028527\n",
      "AccountId_2                     -0.023884\n",
      "AccountId_10                    -0.021586\n",
      "CustomerId_7                    -0.014871\n",
      "SubscriptionId_11               -0.010745\n",
      "CustomerId_10                   -0.010288\n",
      "CustomerId_2                    -0.010247\n",
      "SubscriptionId_8                -0.009600\n",
      "SubscriptionId_7                -0.009546\n",
      "TransactionStartTimeDayofyear   -0.009241\n",
      "TransactionStartTimeMonth       -0.008887\n",
      "Name: FraudResult, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#let's attach the y column to find correlations and after remove\n",
    "train['FraudResult']= y\n",
    "# Find correlations with the target and sort\n",
    "correlations = train.corr()['FraudResult'].sort_values()\n",
    "\n",
    "# Display correlations\n",
    "print('Most Positive Correlations:\\n', correlations.tail(15))\n",
    "print('\\nMost Negative Correlations:\\n', correlations.head(15))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I really don't have much issue with correlation\n",
    "y = train['FraudResult']\n",
    "train = train.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a Validation set\n",
    "\n",
    "We are going to take the last 20,000 values and keep them for validation. We want to use this information to make sure we are checkin properly if our models are doing well on out of sample data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets first mix the train and lable\n",
    "train['FraudResult']= y\n",
    "\n",
    "#choose the last 20k rows to be kept aside\n",
    "valid = train.tail(20000)\n",
    "\n",
    "#bread the validation set into X and y\n",
    "y_valid = valid['FraudResult']\n",
    "X_valid = valid.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75662</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656268</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75663</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656287</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75664</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656326</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75665</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656337</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "75662          0          0          0          0          0          1   \n",
       "75663          0          0          0          0          0          1   \n",
       "75664          0          0          0          1          0          1   \n",
       "75665          0          0          0          1          0          1   \n",
       "\n",
       "       BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  \\\n",
       "75662          1          0          0          1  ...   \n",
       "75663          0          0          0          1  ...   \n",
       "75664          1          0          1          1  ...   \n",
       "75665          0          0          1          0  ...   \n",
       "\n",
       "       TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "75662                              0                             28   \n",
       "75663                              0                             28   \n",
       "75664                              0                             28   \n",
       "75665                              0                             28   \n",
       "\n",
       "       TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "75662                             False                               False   \n",
       "75663                             False                               False   \n",
       "75664                             False                               False   \n",
       "75665                             False                               False   \n",
       "\n",
       "       TransactionStartTimeIs_quarter_end  \\\n",
       "75662                               False   \n",
       "75663                               False   \n",
       "75664                               False   \n",
       "75665                               False   \n",
       "\n",
       "       TransactionStartTimeIs_quarter_start  TransactionStartTimeIs_year_end  \\\n",
       "75662                                 False                            False   \n",
       "75663                                 False                            False   \n",
       "75664                                 False                            False   \n",
       "75665                                 False                            False   \n",
       "\n",
       "       TransactionStartTimeIs_year_start  TransactionStartTimeElapsed  \\\n",
       "75662                              False                   1548656268   \n",
       "75663                              False                   1548656287   \n",
       "75664                              False                   1548656326   \n",
       "75665                              False                   1548656337   \n",
       "\n",
       "       FraudResult  \n",
       "75662            0  \n",
       "75663            0  \n",
       "75664            0  \n",
       "75665            0  \n",
       "\n",
       "[4 rows x 91 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 91)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'FraudResult' in X_valid.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets now take the ones for validation out of the major dataset\n",
    "\n",
    "train  = train.iloc[0:75662, ]\n",
    "\n",
    "y = train['FraudResult']\n",
    "train = train.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((75662, 90), (20000, 90))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, X_valid.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imbalance in data:\n",
    "There is a problem we have with the target variable, we have more of one result than the other, that is we have less fraud cases than the where there was actually fraud.\n",
    "\n",
    "We have to handle this data in a different way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Target variable count')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF3FJREFUeJzt3X/Y3XV93/HnSyJKFSRImmKCwmouvBCnQoRYndtkhkBbg5fVQu2SeUXSDqi129UNnTNXQVpd3dBcs7hMUpKuimjLyBw0S1Hr3BrkRhgISHOLZUkkcGsgDFERfO+P8wk93N/7zn3ukOTckOfjus51vt/35/P9fj/fc905r/P9cU5SVUiS1O85wx6AJGnmMRwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEj7UJIvJfnVAfvuSPLGSdqWJBndt6OTBmc4aJ9K8kjf46dJftg3/64DPJbnJ6kk8w/UNqvqzVX1uQO1vZkiyVVJPjjscWjfmTXsAejZpapeuHs6yd8C76mqv9ybdSWZVVWP76ux7U9JngNQVT8d9likfcEjBx1QSd6Q5MYkDyX5bpLLksxqbbs/6f/zJN8Gvtnqv5hkS1vm40k2J/n1vnX+RpK7k+xM8t+TzGtNX23Pd7cjl7PHjeVnWv3lfbV57WhndpI5Sa5PMtbWfW2SY/r6bk5ycZIbgUeBl/SPLckrknylLTuWZF2Sw8e9JL+Q5Futz5okz5vkdTu2bf97Se5J8pt7eI1fkGR1kq1JdiX5q77X+O1J7myv5V8mWTDutZ/ft54njwZ2n+ZK8oG2L9t3HwkmeS/wduDfttfz85ONTc8choMOtJ8AFwIvBv4B8MvAe8b1+SXgFOC17c34c8DvAHOA77Y2ANr5/fe19cwFbgH+S2t+U3s+oapeWFX/tX8jVfUosAE4t698DrCxqh6k9+/jU8BLgeNb+2XjxvrrwDLgcGDHBPt7MfBzwKuAE4B/M679XODNre21wO+OX0GSQ4DrgP8NvARYAnwgyT+cYHsAq4FXAK8DjgI+CFSSVwFXAucDPwv8FXDt7uAYwMuAtDFcCHwqyQurajXwZ8Al7XV+x4Dr0wxmOOiAqqqvV9VNVfVEVX0b+DQw/k3u0qp6qKp+SO9N/6aq+mJV/QT4GPBgX9/fBD5cVX/T2n8PeGOSuQMO6TM8NRx+rdWoqvur6tqq+mFV7QL+YIKxfrqq7q6qn4w/BVZV36qqL1XVY1W1A/j4BMt/oqq+W1Vjbf3n0vVG4PlV9dG2rr8B/phekD1FkufSC6vfqqod7XX+n1X1ROt/TVV9paoeA36fXuAunPplAnpHR3/Q9vUaoICXT7GMnqG85qADKsmJwL8HTgYOo/c3+L/GddvaN/2S/vmq+mmS7X3tL6P3CfaTfbXHgfnArgGGtBFYl+TVwI+ABcB/a2M9HPgE8E+AI1v/w/Yw1qdI8pK2/C/QO7J4DnDfHpa/l97+jvcy4LgkD/XVDgEmupZzDL3X9NsTtL2kbQOAqnqivZbzgFsn248+Y+OuqTwKvHCyznpm88hBB9p/Br4B/HxVHUHvtEvG9en/qeD76L3RA09e+J3X174V+GdVdWTf47CqunnceibUjja+QO8T+6/R+2T9w9Z8Udv269pYF08x1vH+EPgBcFJb/j0TLH9s3/RL6Z02G28r8K1x+3h4Vb1tgr730QvHn5+g7bv0ggZ48nTVPGA78Bi9U34/09f/5/awb+P5887PMoaDDrTDgV1V9UiSVwLnTdF/A3BakrPaufF/Aczua/8U8MEkJwC0C8lvB6iqH9M7evh7U2zjM/ROuZzbpvvH+ijwUJKj6Z27n47DgUeAh5O8tI19vPcmOaat/yJ611fG+xpAkve1C8ezkvz9JCeP79jCbj3wiSRzkxyS5I0tCD4HvC3Jm9rpp4uA7wMj7YjgduBdbZm3Aq+fxr7ez9Svs55BDAcdaL8DvCfJI8AnmfjN8ElVdR+9N+3VwPfofZK/Hfhxa/8s8B+BP0/yML3TI2/pW8WHgM+3u3PeOslmvkrvNM2LeOqpmo8BR9N7A/0avYvC0/EhetcLdgHX0LtoO95VwJeBLW2//t34Du0N/yx6p6fuBcaAy5n8lM576Z1WuqWN/RIgVXUbsAL4T20dpwNL+66VXAj8Kr1rOmcDX5zGvq4BXtde56umsZxmqPif/eiZpB097AB+uar+etjjkZ6tPHLQjJfkzCQvSvJ8YBW9Uz03D3lY0rOa4aBngjcB3wEeoHcq5G3tVkxJ+4mnlSRJHR45SJI6nrFfgjv66KPruOOOG/YwJOkZ4+abb/5eVc0ZpO8zNhyOO+44RkZGhj0MSXrGSHLv1L16PK0kSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqeMZ+Q/rpOuV31w97CJqBbv7DZcMegjQjeOQgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpY8pwSHJCklv7Hg8neV+So5JsSrKlPc9u/ZNkdZLRJLclOblvXctb/y1JlvfVT0lye1tmdZLsn92VJA1iynCoqrur6jVV9RrgFOBR4BrgIuCGqloA3NDmAc4EFrTHSuBygCRHAauA04BTgVW7A6X1Oa9vuSX7ZO8kSXtluqeVTge+XVX3AkuBda2+Dji7TS8F1lfPZuDIJMcAZwCbqmpnVT0IbAKWtLYjqmpzVRWwvm9dkqQhmG44nAN8tk3Prar72vQOYG6bngds7VtmW6vtqb5tgnpHkpVJRpKMjI2NTXPokqRBDRwOSQ4F3gp8fnxb+8Rf+3BcE6qqNVW1sKoWzpkzZ39vTpIOWtM5cjgT+EZV3d/m72+nhGjPD7T6duDYvuXmt9qe6vMnqEuShmQ64XAuf3dKCWADsPuOo+XAtX31Ze2upUXArnb6aSOwOMnsdiF6MbCxtT2cZFG7S2lZ37okSUMw0H8TmuQFwFuA3+grfwS4OskK4F7gna1+HXAWMErvzqZ3A1TVziSXADe1fhdX1c42fT5wJXAYcH17SJKGZKBwqKofAC8eV/s+vbuXxvct4IJJ1rMWWDtBfQQ4aZCxSJL2P78hLUnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHQOFQ5Ijk3whybeS3JXk9UmOSrIpyZb2PLv1TZLVSUaT3Jbk5L71LG/9tyRZ3lc/JcntbZnVSbLvd1WSNKhBjxw+AfxFVb0CeDVwF3ARcENVLQBuaPMAZwIL2mMlcDlAkqOAVcBpwKnAqt2B0vqc17fckqe3W5Kkp2PKcEjyIuBNwBUAVfVYVT0ELAXWtW7rgLPb9FJgffVsBo5McgxwBrCpqnZW1YPAJmBJazuiqjZXVQHr+9YlSRqCQY4cjgfGgD9OckuSTyd5ATC3qu5rfXYAc9v0PGBr3/LbWm1P9W0T1DuSrEwykmRkbGxsgKFLkvbGIOEwCzgZuLyqXgv8gL87hQRA+8Rf+354T1VVa6pqYVUtnDNnzv7enCQdtAYJh23Atqq6sc1/gV5Y3N9OCdGeH2jt24Fj+5af32p7qs+foC5JGpIpw6GqdgBbk5zQSqcDdwIbgN13HC0Hrm3TG4Bl7a6lRcCudvppI7A4yex2IXoxsLG1PZxkUbtLaVnfuiRJQzBrwH6/BfxpkkOBe4B30wuWq5OsAO4F3tn6XgecBYwCj7a+VNXOJJcAN7V+F1fVzjZ9PnAlcBhwfXtIkoZkoHCoqluBhRM0nT5B3wIumGQ9a4G1E9RHgJMGGYskaf/zG9KSpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgYKhyR/m+T2JLcmGWm1o5JsSrKlPc9u9SRZnWQ0yW1JTu5bz/LWf0uS5X31U9r6R9uy2dc7Kkka3HSOHP5xVb2mqha2+YuAG6pqAXBDmwc4E1jQHiuBy6EXJsAq4DTgVGDV7kBpfc7rW27JXu+RJOlpezqnlZYC69r0OuDsvvr66tkMHJnkGOAMYFNV7ayqB4FNwJLWdkRVba6qAtb3rUuSNASDhkMB/yPJzUlWttrcqrqvTe8A5rbpecDWvmW3tdqe6tsmqHckWZlkJMnI2NjYgEOXJE3XrAH7vbGqtif5WWBTkm/1N1ZVJal9P7ynqqo1wBqAhQsX7vftSdLBaqAjh6ra3p4fAK6hd83g/nZKiPb8QOu+HTi2b/H5rban+vwJ6pKkIZkyHJK8IMnhu6eBxcA3gQ3A7juOlgPXtukNwLJ219IiYFc7/bQRWJxkdrsQvRjY2NoeTrKo3aW0rG9dkqQhGOS00lzgmnZ36SzgM1X1F0luAq5OsgK4F3hn638dcBYwCjwKvBugqnYmuQS4qfW7uKp2tunzgSuBw4Dr20OSNCRThkNV3QO8eoL694HTJ6gXcMEk61oLrJ2gPgKcNMB4JUkHgN+QliR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkjoHDIckhSW5J8sU2f3ySG5OMJvlckkNb/XltfrS1H9e3jve3+t1JzuirL2m10SQX7bvdkyTtjekcOfw2cFff/EeBy6rq5cCDwIpWXwE82OqXtX4kORE4B3glsAT4oxY4hwCfBM4ETgTObX0lSUMyUDgkmQ/8IvDpNh/gzcAXWpd1wNltemmbp7Wf3vovBa6qqh9X1XeAUeDU9hitqnuq6jHgqtZXkjQkgx45fBz4V8BP2/yLgYeq6vE2vw2Y16bnAVsBWvuu1v/J+rhlJqt3JFmZZCTJyNjY2IBDlyRN15ThkOSXgAeq6uYDMJ49qqo1VbWwqhbOmTNn2MORpGetWQP0eQPw1iRnAc8HjgA+ARyZZFY7OpgPbG/9twPHAtuSzAJeBHy/r75b/zKT1SVJQzDlkUNVvb+q5lfVcfQuKH+pqt4FfBn4ldZtOXBtm97Q5mntX6qqavVz2t1MxwMLgK8DNwEL2t1Ph7ZtbNgneydJ2iuDHDlM5l8DVyX5MHALcEWrXwH8SZJRYCe9N3uq6o4kVwN3Ao8DF1TVEwBJLgQ2AocAa6vqjqcxLknS0zStcKiqrwBfadP30LvTaHyfHwHvmGT5S4FLJ6hfB1w3nbFIkvYfvyEtSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdU4ZDkucn+XqS/5PkjiS/1+rHJ7kxyWiSzyU5tNWf1+ZHW/txfet6f6vfneSMvvqSVhtNctG+301J0nQMcuTwY+DNVfVq4DXAkiSLgI8Cl1XVy4EHgRWt/wrgwVa/rPUjyYnAOcArgSXAHyU5JMkhwCeBM4ETgXNbX0nSkEwZDtXzSJt9bnsU8GbgC62+Dji7TS9t87T205Ok1a+qqh9X1XeAUeDU9hitqnuq6jHgqtZXkjQkA11zaJ/wbwUeADYB3wYeqqrHW5dtwLw2PQ/YCtDadwEv7q+PW2ayuiRpSAYKh6p6oqpeA8yn90n/Fft1VJNIsjLJSJKRsbGxYQxBkg4K07pbqaoeAr4MvB44Msms1jQf2N6mtwPHArT2FwHf76+PW2ay+kTbX1NVC6tq4Zw5c6YzdEnSNAxyt9KcJEe26cOAtwB30QuJX2ndlgPXtukNbZ7W/qWqqlY/p93NdDywAPg6cBOwoN39dCi9i9Yb9sXOSZL2zqypu3AMsK7dVfQc4Oqq+mKSO4GrknwYuAW4ovW/AviTJKPATnpv9lTVHUmuBu4EHgcuqKonAJJcCGwEDgHWVtUd+2wPJUnTNmU4VNVtwGsnqN9D7/rD+PqPgHdMsq5LgUsnqF8HXDfAeCVJB4DfkJYkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUMWU4JDk2yZeT3JnkjiS/3epHJdmUZEt7nt3qSbI6yWiS25Kc3Leu5a3/liTL++qnJLm9LbM6SfbHzkqSBjPIkcPjwL+sqhOBRcAFSU4ELgJuqKoFwA1tHuBMYEF7rAQuh16YAKuA04BTgVW7A6X1Oa9vuSVPf9ckSXtrynCoqvuq6htt+v8BdwHzgKXAutZtHXB2m14KrK+ezcCRSY4BzgA2VdXOqnoQ2AQsaW1HVNXmqipgfd+6JElDMK1rDkmOA14L3AjMrar7WtMOYG6bngds7VtsW6vtqb5tgvpE21+ZZCTJyNjY2HSGLkmahoHDIckLgT8D3ldVD/e3tU/8tY/H1lFVa6pqYVUtnDNnzv7enCQdtAYKhyTPpRcMf1pVf97K97dTQrTnB1p9O3Bs3+LzW21P9fkT1CVJQzLI3UoBrgDuqqr/0Ne0Adh9x9Fy4Nq++rJ219IiYFc7/bQRWJxkdrsQvRjY2NoeTrKobWtZ37okSUMwa4A+bwD+KXB7kltb7QPAR4Crk6wA7gXe2dquA84CRoFHgXcDVNXOJJcAN7V+F1fVzjZ9PnAlcBhwfXtIkoZkynCoqq8Bk33v4PQJ+hdwwSTrWgusnaA+Apw01VgkSQeG35CWJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqSOKcMhydokDyT5Zl/tqCSbkmxpz7NbPUlWJxlNcluSk/uWWd76b0myvK9+SpLb2zKrk0z2X5JKkg6QQY4crgSWjKtdBNxQVQuAG9o8wJnAgvZYCVwOvTABVgGnAacCq3YHSutzXt9y47clSTrApgyHqvoqsHNceSmwrk2vA87uq6+vns3AkUmOAc4ANlXVzqp6ENgELGltR1TV5qoqYH3fuiRJQ7K31xzmVtV9bXoHMLdNzwO29vXb1mp7qm+boD6hJCuTjCQZGRsb28uhS5Km8rQvSLdP/LUPxjLIttZU1cKqWjhnzpwDsUlJOijtbTjc304J0Z4faPXtwLF9/ea32p7q8yeoS5KGaG/DYQOw+46j5cC1ffVl7a6lRcCudvppI7A4yex2IXoxsLG1PZxkUbtLaVnfuiRJQzJrqg5JPgv8I+DoJNvo3XX0EeDqJCuAe4F3tu7XAWcBo8CjwLsBqmpnkkuAm1q/i6tq90Xu8+ndEXUYcH17SJKGaMpwqKpzJ2k6fYK+BVwwyXrWAmsnqI8AJ001DknSgeM3pCVJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1zJhwSLIkyd1JRpNcNOzxSNLBbEaEQ5JDgE8CZwInAucmOXG4o5Kkg9esYQ+gORUYrap7AJJcBSwF7hzqqKQh+b8Xv2rYQ9AM9NIP3X7AtjVTwmEesLVvfhtw2vhOSVYCK9vsI0nuPgBjOxgcDXxv2IOYCfKx5cMegrr8+9xtVZ7uGl42aMeZEg4Dqao1wJphj+PZJslIVS0c9jikifj3ORwz4poDsB04tm9+fqtJkoZgpoTDTcCCJMcnORQ4B9gw5DFJ0kFrRpxWqqrHk1wIbAQOAdZW1R1DHtbBxFN1msn8+xyCVNWwxyBJmmFmymklSdIMYjhIkjoMh4OcP1uimSrJ2iQPJPnmsMdyMDIcDmL+bIlmuCuBJcMexMHKcDi4PfmzJVX1GLD7Z0ukoauqrwI7hz2Og5XhcHCb6GdL5g1pLJJmEMNBktRhOBzc/NkSSRMyHA5u/myJpAkZDgexqnoc2P2zJXcBV/uzJZopknwW+GvghCTbkqwY9pgOJv58hiSpwyMHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLU8f8Bh4cyt2WtBBIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#i want to visualize the imbalance in the target variable\n",
    "x=y.value_counts().values\n",
    "sns.barplot([0,1],x)\n",
    "plt.title('Target variable count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Smote: Oversampling data:\n",
    "\n",
    "`SMOTE (Synthetic Minority Oversampling TEchnique)` consists of synthesizing elements for the minority class, based on those that already exist. It works randomly picking a point from the minority class and computing the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60529, 90), (15133, 90), (60529,), (15133,))"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#First we are going to separate the target variable from the rest of the observations and split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x = train\n",
    "\n",
    "#split the data to prevent overfitting\n",
    "X_train,X_test,y_train,y_test=train_test_split(x,y,test_size=.2,random_state=1)\n",
    "\n",
    "#let's see the shape of the data\n",
    "X_train.shape,X_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#first we are going to filter out the different results from the label in the main label\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "smote = SMOTE(random_state=27,ratio='minority')\n",
    "X_train, y_train = smote.fit_sample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((120846, 90), (120846,))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's see the shape\n",
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After OverSampling, the shape of train_X: (120846, 90)\n",
      "After OverSampling, the shape of train_y: (120846,) \n",
      "\n",
      "After OverSampling, counts of label '1': 60423\n",
      "After OverSampling, counts of label '0': 60423\n"
     ]
    }
   ],
   "source": [
    "#let's check and see the number of os and ones\n",
    "print('After OverSampling, the shape of train_X: {}'.format(X_train.shape))\n",
    "print('After OverSampling, the shape of train_y: {} \\n'.format(y_train.shape))\n",
    "\n",
    "print(\"After OverSampling, counts of label '1': {}\".format(sum(y_train==1)))\n",
    "print(\"After OverSampling, counts of label '0': {}\".format(sum(y_train==0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's take a test run at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.5/site-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9999751750161362"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now we have something to put in the model\n",
    "m = RandomForestClassifier(n_jobs=-1)\n",
    "m.fit(X_train, y_train)\n",
    "m.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Score Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I want to write a function that I will call to evaluate my model, I am going to evaluate based on ROC_AUC and score.\n",
    "\n",
    "#when you build a model you want to know how it is functioning. And there are functions that are pre_built in scikit learn \n",
    "#to help you evaluate the model, so I built these in the functions below\n",
    "\n",
    "def print_score(m,md):\n",
    "    \n",
    "    # Used to ignore warnings generated from StackingCVClassifier\n",
    "    import warnings\n",
    "    warnings.simplefilter('ignore')\n",
    "\n",
    "    #Call the classification metrics we want to use\n",
    "    from sklearn.metrics import roc_auc_score,auc,f1_score,roc_curve\n",
    "    \n",
    "    \n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, m)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "    \n",
    "    f1 = f1_score(y_test,m)\n",
    "    roc = roc_auc_score(y_test,m)\n",
    "    #oob = md.oob_score_ #I added the oob score\n",
    "    \n",
    "    #let's see how the model does on validation data\n",
    "    pred = md.predict(X_valid)\n",
    "    f2 = f1_score(y_valid,pred)\n",
    "    \n",
    "    pred_3 = md.predict(X_train)\n",
    "    f3 = f1_score(y_train,pred_3)\n",
    "    \n",
    "    res = [\"roc_auc-->\",roc_auc,\"F1_score-->\",f1,\"roc_auc_score-->\",roc]\n",
    "    print(res)\n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on out of sample data\")\n",
    "    \n",
    "    valid = ['Valid_data_F1_score-->',f2]\n",
    "    \n",
    "    print(valid)\n",
    "    \n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on train data\")\n",
    "    \n",
    "    train_ = ['Train_data_F1_score-->',f3]\n",
    "    \n",
    "    print(train_)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('--'*10)\n",
    "    print('Scores on the Validation Data according to the Confusion_matrix')\n",
    "    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    tn, fp, fn, tp = confusion_matrix(y_valid,pred).ravel()\n",
    "    \n",
    "    # Error rate : \n",
    "    err_rate = (fp + fn) / (tp + tn + fn + fp)\n",
    "    print(\"Error rate  : \", err_rate)\n",
    "\n",
    "    # Accuracy : \n",
    "    acc_ = (tp + tn) / (tp + tn + fn + fp)\n",
    "    print(\"Accuracy  : \", acc_)\n",
    "    \n",
    "    \n",
    "    #precision\n",
    "    preci_ = tp / (tp + fp)\n",
    "    print(\"Precision  : \", preci_)\n",
    "\n",
    "    # Sensitivity : \n",
    "    sens_ = tp / (tp + fn)\n",
    "    print(\"Recall  : \", sens_)\n",
    "\n",
    "    # Specificity \n",
    "    sp_ = tn / (tn + fp)\n",
    "    print(\"Specificity  : \", sens_)\n",
    "\n",
    "    # False positive rate (FPR)\n",
    "    FPR = fp / (tn + fp)\n",
    "    print(\"False positive rate  : \", FPR)\n",
    "    \n",
    "    # False positive rate (FPR)\n",
    "    TPR = tp / (tp + fn)\n",
    "    print(\"True positive rate  : \", TPR)\n",
    "    \n",
    "    print('--'*10)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('Drawing the Confusion Matrix on Test_data')\n",
    "    #from sklearn.metrics import confusion_matrix\n",
    "    \n",
    "    plt.figure(figsize=(7,7))\n",
    "    cm_2 = confusion_matrix(y_test,m)\n",
    "    sns.heatmap(cm_2,annot=True,fmt=\"d\")\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Models\n",
    "\n",
    "Here we are going to use crossvalidation to run baseline models and set a stndard of what we are going to do we are going to use three models:\n",
    "\n",
    "1- Random Forest Classifier\n",
    "2- Light Gradient Boosting Machine\n",
    "3- X Gradient Bosting Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#thee are the necessary libraries we will need\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from timeit import default_timer as timer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of  10 | elapsed:    5.9s remaining:   13.8s\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    6.5s finished\n"
     ]
    }
   ],
   "source": [
    "#Random Forest Classifier with Cross val\n",
    "\n",
    "rf_ = RandomForestClassifier()\n",
    "\n",
    "start = timer()\n",
    "cv = cross_val_score(estimator = rf_, X = X_train, \n",
    "                     y = y_train, \n",
    "                     verbose = 2, n_jobs = -1, \n",
    "                     cv = 10, scoring = 'f1')\n",
    "cv_time = timer() - start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline model 10 fold cv F1_score: 0.99948\n",
      "Baseline model eval time: 6.48 seconds.\n"
     ]
    }
   ],
   "source": [
    "print('Baseline model 10 fold cv F1_score: {:.5f}'.format(np.mean(cv)))\n",
    "print('Baseline model eval time: {:.2f} seconds.'.format(cv_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Light Gradient Boosting Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of  10 | elapsed:    9.9s remaining:   23.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline model 10 fold cv F1_score: 0.99977\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:   11.8s finished\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "model_lgb = lgb.LGBMClassifier()\n",
    "\n",
    "start = timer()\n",
    "cv = cross_val_score(estimator = model_lgb, X = X_train, \n",
    "                     y = y_train, \n",
    "                     verbose = 2, n_jobs = -1, \n",
    "                     cv = 10, scoring = 'f1')\n",
    "cv_time = timer() - start\n",
    "\n",
    "\n",
    "print('Baseline model 10 fold cv F1_score: {:.5f}'.format(np.mean(cv)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### X Gradient Boosting Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xgboost\n",
      "  Downloading https://files.pythonhosted.org/packages/c1/24/5fe7237b2eca13ee0cfb100bec8c23f4e69ce9df852a64b0493d49dae4e0/xgboost-0.90-py2.py3-none-manylinux1_x86_64.whl (142.8MB)\n",
      "\u001b[K    100% |████████████████████████████████| 142.8MB 9.6kB/s \n",
      "\u001b[?25hRequirement already satisfied: scipy in /home/jupyter/.local/lib/python3.5/site-packages (from xgboost)\n",
      "Requirement already satisfied: numpy in /home/jupyter/.local/lib/python3.5/site-packages (from xgboost)\n",
      "Installing collected packages: xgboost\n",
      "Successfully installed xgboost-0.90\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#%pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost.sklearn import XGBRFClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline model 10 fold cv F1_score: 0.99144\n"
     ]
    }
   ],
   "source": [
    "xgb = XGBRFClassifier()\n",
    "\n",
    "start = timer()\n",
    "cv = cross_val_score(estimator = xgb, X = X_train, \n",
    "                     y = y_train, \n",
    "                     verbose = 0, n_jobs = -1, \n",
    "                     cv = 10, scoring = 'f1')\n",
    "cv_time = timer() - start\n",
    "\n",
    "\n",
    "print('Baseline model 10 fold cv F1_score: {:.5f}'.format(np.mean(cv)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Tuning\n",
    "\n",
    "From here on, we embark in tuning the above models such that they will be able to generalize properly, what we have above maybe overfitted models that may not generalize well on out of sample data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8min 24s, sys: 292 ms, total: 8min 24s\n",
      "Wall time: 32.4 s\n",
      "['roc_auc-->', 0.8094245506566841, 'F1_score-->', 0.7027027027027026, 'roc_auc_score-->', 0.8094245506566841]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.0]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 1.0]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.0033\n",
      "Accuracy  :  0.9967\n",
      "Precision  :  nan\n",
      "Recall  :  0.0\n",
      "Specificity  :  0.0\n",
      "False positive rate  :  0.0\n",
      "True positive rate  :  0.0\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGYpJREFUeJzt3X+4lWWZ6PHvLQgiMyMixiB6PKRYqTMaIWrX6DiSgNYIdhyzmYpjnpgp7NfMlDp5DqnVpVelnSZHDwaFWiCX4sj4G/FXTSXQET2ikjtNYftbRBNN2u7n/LFfbEXCBlx7r5v9fj9e78Vaz3rWWs9SrnV73++9njdKKUiS1Go7tHoBkiSBAUmSlIQBSZKUggFJkpSCAUmSlIIBSZKUggFJkpSCAUmSlIIBSZJqKCJmR8QzEXF/w9iXI6I9IpZXx3ENj50ZEW0RsTIiJjaMT6rG2iLijIbxURFxdzV+ZUQM6HZNPb1Tw2+fe8StINRrBu1xRKuXoJrpWN8ezXqtZn5f7jjs7ZtdV0QcCbwMXFZKObAa+zLwcinlGxvN3R+YC4wD9gBuBfarHv4FcAywGlgKfLiU8kBEzAcWlFLmRcQlwL2llIs3tyYzJEmqoVLKXcCaLZw+GZhXSnmtlPIo0EZXcBoHtJVSHimlrAfmAZMjIoCjgauq588BpnT3JgYkScqi8/XmHdvutIi4ryrp7VqNjQRWNcxZXY1tanw3YG0ppWOj8c0yIElSHxQR0yJiWcMxbQuedjGwD3Aw8CTwzR5d5Eb69+abSZI2o3Q276VKmQnM3MrnPL3hdkRcClxX3W0H9mqYumc1xibGnweGRET/KktqnL9JZkiSlEVnZ/OObRARIxrungBs6MBbCJwcEQMjYhQwGlhCVxPD6KqjbgBwMrCwdHXL3Q6cWD1/KnBtd+9vhiRJNRQRc4GjgGERsRqYARwVEQcDBfgV8PcApZQVVdfcA0AHML2U8nr1OqcBNwP9gNmllBXVW5wOzIuIrwD3ALO6XZNt3+pLbPtWb2tm2/f6J1Y07ftywB4HNG1dvcUMSZKy2MZSW1/hOSRJUgpmSJKURRO77LZHBiRJyuKt/aB1u2fJTpKUghmSJGVhyU6SlIJddpIktZ4ZkiQlUSzZSZJSsGQnSVLrmSFJUhaW7CRJKfjDWEmSWs8MSZKysGQnSUrBLjtJklrPDEmSsrBkJ0lKwZKdJEmtZ4YkSUmUUu/fIRmQJCmLmp9DsmQnSUrBDEmSsqh5U4MBSZKyqHnJzoAkSVm4uaokSa1nhiRJWViykySlUPOmBkt2kqQUzJAkKQtLdpKkFCzZSZLUemZIkpRFzTMkA5IkJVH33b4t2UmSUjBDkqQsLNlJklKoedu3JTtJUgpmSJKUhSU7SVIKluwkSWo9MyRJysKSnSQpBUt2kiS1nhmSJGVhyU6SlELNA5IlO0lSCmZIkpRFzZsaDEiSlIUlO0mSWs8MSZKysGQnSUrBkp0kSa1nhiRJWViykySlYMlOkqTWM0OSpCxqniEZkCQpi1JavYKWsmQnSUrBDEmSsrBkJ0lKoeYByZKdJCkFMyRJysIfxkqSUrBkJ0lS65khSVIW/g5JkpRCZ2fzjm5ExOyIeCYi7m8Y+3pEPBQR90XENRExpOGxMyOiLSJWRsTEhvFJ1VhbRJzRMD4qIu6uxq+MiAHdrcmAJEn19H1g0kZji4ADSyl/DvwCOBMgIvYHTgYOqJ7zbxHRLyL6ARcBxwL7Ax+u5gKcD1xYStkXeAE4tbsFGZAkKYtezJBKKXcBazYau6WU0lHd/RmwZ3V7MjCvlPJaKeVRoA0YVx1tpZRHSinrgXnA5IgI4Gjgqur5c4Ap3a3Jc0iSlEWutu+PA1dWt0fSFaA2WF2NAazaaPxQYDdgbUNwa5y/SWZIktQHRcS0iFjWcEzbiud+CegAftBzK/xDZkiSlETpbF6XXSllJjBza58XEf8d+AAwvpQ32v7agb0apu1ZjbGJ8eeBIRHRv8qSGudvkhmSJGXRi+eQ3kxETAK+CBxfSnml4aGFwMkRMTAiRgGjgSXAUmB01VE3gK7Gh4VVILsdOLF6/lTg2u7e34AkSTUUEXOBnwLviIjVEXEq8B3gj4FFEbE8Ii4BKKWsAOYDDwA3AdNLKa9X2c9pwM3Ag8D8ai7A6cA/RkQbXeeUZnW7ptLDP8T67XOP1PuXXupVg/Y4otVLUM10rG+PZr3WKxd/umnflzt/8l+btq7e4jkkScqiieeQtkeW7CRJKZghSVIWNd/t24AkSVkYkCRJKbjbtyRJrWeGJElZWLJTTzrraxdw138uYeiuQ/j3Ky4B4KJZV3D1wpvYdcguAHz276dy5HvHsfbFl/j8l77K/Q/9ginHHsOX/ulTb7zOioce5qyvXsBvXnuNIw4/hDM/9w9EBA89/Ajnfv1feeXV37DHiLdx/owv8keDB7fks2r7M3DgQO647WoGDBxI//79WLDges4+55utXlZ92fatnjTluGO45IKv/MH4Rz80havnXMTVcy7iyPeOA2DAgAF8+hMf5Z+n/48/mH/uN77Dl0//DDdcOYvHVz/Bj3+2DIAZ532Lz33yFK65/GLGH/levveDq3v2A6lPee2113jfhJN4z9hjeM/YCUyccBSHjhvT6mWpproNSBHxzog4PSK+XR2nR8S7emNxfcHYg/+MXf7kj7do7s6DdmLMQQcycMDvX1jx2efWsG7dKxx04LuICI6fNJ7bfvRTAB5b1c7Yg/8MgMMPGcOiO3/c3A+gPm/duq4ty3bcsT/9d9yRnt69RZtROpt3bIc2G5Ai4nS6LrgUdG2kt6S6PbfxUrXaenOv/g9O+NgnOetrF/DiS7/e7Nynn32O4W8b9sb94bsP4+lnnwdgn1F7vxGcbrn9Rzz19HM9t2j1STvssAPLlt7Ck+33sXjxXSxZek+rl1RfnaV5x3aouwzpVOCQUsp5pZQrquM8uq4SuMnL0TZeh+O7l81t5nr7hA+d8H5unD+bq79/EbvvNpSvf+fSbX6tc//l88xbcB0nffzTrHvlVXbc0dOC2jqdnZ2MPWQCe48ayyFj380BB7yj1UtSTXX37dUJ7AE8ttH4iOqxN9V4HQ43V/1Dw4bu+sbtE48/lulfmLHZ+cN3H8bTz/wu83n62ecYvvtuALx977249FtfA+BXj6/mrp8s6YEVqw5efPEl7rjzP5k44ShWrFjZ6uXUUql5l113GdLngMURcWNEzKyOm4DFwGd7fnl907PP/e4y9ovv/An7vn3vzc7ffdhQBg/emXvvf5BSCgtvWsxf/cVhADz/wlqg6/9y/8+ceZw05bieW7j6nGHDhrLLLn8CwE477cT7xh/JypW/bPGqaqzmJbvNZkillJsiYj+6SnQbrofeDiwtpbze04vrC74w4zyW3nMfa9e+xPgpH+FTp36Upffcx8qHH4GAkX86nBlf/Mwb8yf8t6m8vO4VftvRwW0/+gkzL/wq+4zam7P+afrv2r4PO4QjDj8EgBsW3cG8BdcB8L6/fC8nvH9CSz6ntk8jRgxn9qxv0a/fDuywww5cddV/cP0Nt7Z6Waopr4ekPsXrIam3NfN6SOu+8pGmfV8OPusKr4ckSdpG22mprVn8YawkKQUzJEnKouZddgYkScrCkp0kSa1nhiRJWWyne9A1iwFJkrKwZCdJUuuZIUlSEnXfy86AJElZWLKTJKn1zJAkKYuaZ0gGJEnKouZt35bsJEkpmCFJUhaW7CRJGZSaByRLdpKkFMyQJCmLmmdIBiRJyqLmOzVYspMkpWCGJElZWLKTJKVQ84BkyU6SlIIZkiQlUUq9MyQDkiRlYclOkqTWM0OSpCxqniEZkCQpCfeykyQpATMkScqi5hmSAUmSsqj3VnaW7CRJOZghSVISdW9qMCBJUhY1D0iW7CRJKZghSVIWNW9qMCBJUhJ1P4dkyU6SlIIZkiRlYclOkpSBJTtJkhIwQ5KkLCzZSZIyKAYkSVIKNQ9InkOSJKVghiRJSViykyTlUPOAZMlOkpSCGZIkJWHJTpKUQt0DkiU7SVIKBiRJSqJ0Nu/YEhHx2Yi4PyJWRMTnqrGhEbEoIh6u/ty1Go+I+HZEtEXEfRExpuF1plbzH46Iqdv6+Q1IkpRFieYd3YiIA4FPAOOAg4APRMS+wBnA4lLKaGBxdR/gWGB0dUwDLq5eZygwAzi0eq0ZG4LY1jIgSVI9vQu4u5TySimlA7gT+CAwGZhTzZkDTKluTwYuK11+BgyJiBHARGBRKWVNKeUFYBEwaVsWZECSpCSaWbKLiGkRsazhmLbR290PHBERu0XEzsBxwF7A8FLKk9Wcp4Dh1e2RwKqG56+uxjY1vtXsspOkJEpn96W2LX6tUmYCMzfz+IMRcT5wC7AOWA68vtGcEhG9dpEmMyRJqqlSyqxSyntKKUcCLwC/AJ6uSnFUfz5TTW+nK4PaYM9qbFPjW82AJElJtKDL7m3Vn/+FrvNHPwQWAhs65aYC11a3FwIfq7rtDgNerEp7NwMTImLXqplhQjW21SzZSVISZQu645rs6ojYDfgtML2UsjYizgPmR8SpwGPASdXcG+g6z9QGvAKc0rXmsiYizgWWVvPOKaWs2ZbFGJAkqaZKKUe8ydjzwPg3GS/A9E28zmxg9ltdjwFJkpKo+9ZBBiRJSqKZXXbbI5saJEkpmCFJUhKl137xk5MBSZKSsGQnSVICZkiSlETdMyQDkiQlUfdzSJbsJEkpmCFJUhKW7CRJKbRgL7tULNlJklIwQ5KkJNzLTpKUQqclO0mSWs8MSZKSqHtTgwFJkpKoe9u3JTtJUgpmSJKURN23DjIgSVISluwkSUrADEmSkqj775AMSJKURN3bvi3ZSZJSMEOSpCTsspMkpVD3c0iW7CRJKZghSVISdW9qMCBJUhJ1P4dkyU6SlEKPZ0iD9jiip99CkvqEujc1WLKTpCTqfg7Jkp0kKQUzJElKwpKdJCmFmjfZGZAkKYu6Z0ieQ5IkpWCGJElJ1L3LzoAkSUl0tnoBLWbJTpKUghmSJCVRsGQnSUqgs+Z935bsJEkpmCFJUhKdluwkSRnU/RySJTtJUgpmSJKURN1/h2RAkqQkLNlJkpSAGZIkJWHJTpKUQt0DkiU7SVIKZkiSlETdmxoMSJKURGe945ElO0lSDmZIkpSEe9lJklKo+dUnLNlJknIwQ5KkJOr+OyQDkiQl0Rn1PodkyU6SlIIZkiQlUfemBgOSJCVR93NIluwkSSmYIUlSEnXfOsiAJElJ1H2nBkt2klRTETEkIq6KiIci4sGIODwihkbEooh4uPpz12puRMS3I6ItIu6LiDENrzO1mv9wREzd1vUYkCQpidLEYwv9b+CmUso7gYOAB4EzgMWllNHA4uo+wLHA6OqYBlwMEBFDgRnAocA4YMaGILa1DEiSlERnNO/oTkTsAhwJzAIopawvpawFJgNzqmlzgCnV7cnAZaXLz4AhETECmAgsKqWsKaW8ACwCJm3L5zcgSVI9jQKeBb4XEfdExHcjYjAwvJTyZDXnKWB4dXsksKrh+aursU2NbzUDkiQl0dnEIyKmRcSyhmPaRm/XHxgDXFxKeTewjt+V5wAopWxlBfCtsctOkpJo5jd/KWUmMHMzU1YDq0spd1f3r6IrID0dESNKKU9WJblnqsfbgb0anr9nNdYOHLXR+B3bsmYzJEmqoVLKU8CqiHhHNTQeeABYCGzolJsKXFvdXgh8rOq2Owx4sSrt3QxMiIhdq2aGCdXYVjNDkqQkWvDD2E8DP4iIAcAjwCl0JSrzI+JU4DHgpGruDcBxQBvwSjWXUsqaiDgXWFrNO6eUsmZbFmNAkqQkensvu1LKcmDsmzw0/k3mFmD6Jl5nNjD7ra7Hkp0kKQUzJElKou67fRuQJCmJUu+t7CzZSZJyMEOSpCQs2UmSUqh7QLJkJ0lKwQxJkpLotU3jkjIgSVISdb+EuSU7SVIKZkiSlETdmxoMSJKURN0DkiU7SVIKZkiSlIRddpKkFOreZWdAkqQkPIckSVICZkiSlITnkCRJKXTWPCRZspMkpWCGJElJ1L2pwYAkSUnUu2BnyU6SlIQZkiQlYclOkpRC3XdqsGQnSUrBDEmSkqj775AMSJKURL3DkSU7SVISZkiSlIRddpKkFOp+DsmSnSQpBTMkSUqi3vmRAUmS0qj7OSRLdpKkFMyQJCmJujc1GJAkKYl6hyNLdpKkJMyQJCmJujc1GJAkKYlS86KdJTtJUgpmSJKUhCU7SVIKdW/7tmQnSUrBDEmSkqh3fmRAkqQ0LNkpnc9+5hPcu/w2lt+zmCsuv4iBAwe2eknqQy6d+U2eWH0vy+9Z/MbY2V/+Av/354tYtvQWbrz+h4wYMbyFK1RdGZCS2WOPP+W06R/n0MOO4+B3j6dfv3586KTJrV6W+pDLLpvP+z/wd7839o1vXsyY9xzD2EMmcP0Nt3LWlz7fotXVW2cTj+2RASmh/v37M2jQTvTr14+dBw3iySefavWS1If86Md3s+aFtb839utfv/zG7cGDd6aUepeOWqU08Z/t0TafQ4qIU0op32vmYgRPPPEUF1x4CY/+cgmvvvobFt16J4tuvavVy1INnHvO6Xzk707kxZde4n3H/E2rl6MaeisZ0tlNW4XeMGTILhz/1xPZd7/D2GvvMQwevDN/+7cfbPWyVAP/83+dz6h9DmHu3GuY/qlTWr2cWrJktxkRcd8mjv8HbPKsZ0RMi4hlEbGss3Nd0xfdl40ffwSP/upxnntuDR0dHVzz7zdy+GFjW70s1cgP5y7ghBOOa/UyasmS3eYNByYCL2w0HsBPNvWkUspMYCZA/wEjt89/My2y6vF2Dj10DIMG7cSrr/6Go//qL/j5z+9t9bLUx+277yja2h4F4Pi/nsjKlb9s8YpUR90FpOuAPyqlLN/4gYi4o0dWVHNLlt7DggXXs3TJzXR0dLB8+Qou/e4PWr0s9SFXXH4Rf3nk4QwbNpRfPbKMs8/5BsceezT77bcPnZ2dPP54O5+afkarl1lL22uprVmip7tpzJAk9WUd69ujWa/10b0/2LTvy8sfW9C0dfUW274lSSm4dZAkJVH3cpIBSZKScC87SZISMEOSpCS2198PNYsBSZKSqHvbtyU7SVIKZkiSlETdmxoMSJKURN3PIVmykySlYIYkSUnUvanBgCRJSdT9Sr2W7CSphiJip4hYEhH3RsSKiDi7Gh8VEXdHRFtEXBkRA6rxgdX9turx/9rwWmdW4ysjYuK2rsmAJElJdFKadmyB14CjSykHAQcDkyLiMOB84MJSyr50XQvv1Gr+qcAL1fiF1TwiYn/gZOAAYBLwbxHRb1s+vwFJkpLozUuYly4vV3d3rI4CHA1cVY3PAaZUtydX96keHx8RUY3PK6W8Vkp5FGgDxm3tZwcDkiSl0cxLmEfEtIhY1nBM2/j9IqJfRCwHngEWAb8E1pZSOqopq4GR1e2RwCqA6vEXgd0ax9/kOVvFpgZJ6oNKKTOBmd3MeR04OCKGANcA7+yNtW2KAUmSkmjVTg2llLURcTtwODAkIvpXWdCeQHs1rR3YC1gdEf2BXYDnG8Y3aHzOVrFkJ0lJlFKadnQnInavMiMiYhBwDPAgcDtwYjVtKnBtdXthdZ/q8dtK1xstBE6uuvBGAaOBJdvy+c2QJKmeRgBzqo64HYD5pZTrIuIBYF5EfAW4B5hVzZ8FXB4RbcAaujrrKKWsiIj5wANABzC9KgVutejpH2L1HzCy3r/0ktSndaxvj2a91sS9jm3a9+XNq25s2rp6ixmSJCXh5qqSJCVghiRJSXg9JElSCm6uKklSAmZIkpSEJTtJUgp22UmSlIAZkiQl0VnzpgYDkiQlUe9wZMlOkpSEGZIkJWGXnSQphboHJEt2kqQUzJAkKYm6bx1kQJKkJCzZSZKUgBmSJCVR962DDEiSlETdzyFZspMkpWCGJElJ1L2pwYAkSUlYspMkKQEzJElKwpKdJCmFurd9W7KTJKVghiRJSXjFWElSCpbsJElKwAxJkpKwZCdJSsGSnSRJCZghSVISluwkSSlYspMkKQEzJElKwpKdJCkFS3aSJCVghiRJSZTS2eoltJQBSZKSqPv1kCzZSZJSMEOSpCSKXXaSpAws2UmSlIAZkiQlYclOkpRC3XdqsGQnSUrBDEmSkqj71kEGJElKwnNIkqQUbPuWJCkBMyRJSsKSnSQpBdu+JUlKwAxJkpKwZCdJSsEuO0mSEjBDkqQkLNlJklKwy06SpATMkCQpCTdXlSSlYMlOkqQEzJAkKQm77CRJKdT9HJIlO0lSCgYkSUqilNK0Y0tExKSIWBkRbRFxRg9/vG5ZspOkJHrzHFJE9AMuAo4BVgNLI2JhKeWBXlvERsyQJKmexgFtpZRHSinrgXnA5FYuyIAkSUmUJh5bYCSwquH+6mqsZXq8ZNexvj16+j36ooiYVkqZ2ep1qD78O9d6zfy+jIhpwLSGoZnZ//uaIeU1rfspUlP5d64PKaXMLKWMbTg2DkbtwF4N9/esxlrGgCRJ9bQUGB0RoyJiAHAysLCVC7LLTpJqqJTSERGnATcD/YDZpZQVrVyTASmv1LVe9Un+nauZUsoNwA2tXscGUfe9kyRJOXgOSZKUggEpoWzbeajviojZEfFMRNzf6rVIBqRkGrbzOBbYH/hwROzf2lWpD/s+MKnVi5DAgJRRuu081HeVUu4C1rR6HRIYkDJKt52HJPUGA5IkKQUDUj7ptvOQpN5gQMon3XYektQbDEjJlFI6gA3beTwIzG/1dh7quyJiLvBT4B0RsToiTm31mlRf7tQgSUrBDEmSlIIBSZKUggFJkpSCAUmSlIIBSZKUggFJkpSCAUmSlIIBSZKUwv8HTexU0Ka8jM0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml = RandomForestClassifier(n_jobs=-1, n_estimators=1060, max_features='sqrt',random_state=20)\n",
    "%time ml.fit(X_train, y_train)\n",
    "pred_d = ml.predict(X_test)\n",
    "print_score(pred_d,ml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#classifiers\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Used to ignore warnings generated from StackingCVClassifier\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV,RandomizedSearchCV\n",
    "from sklearn.metrics import roc_curve, auc,f1_score\n",
    "\n",
    "#we are going to use cross validation\n",
    "from sklearn.model_selection import KFold, RepeatedStratifiedKFold\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': [True],\n",
      " 'max_depth': [4,\n",
      "               4,\n",
      "               5,\n",
      "               6,\n",
      "               7,\n",
      "               8,\n",
      "               9,\n",
      "               9,\n",
      "               10,\n",
      "               11,\n",
      "               12,\n",
      "               13,\n",
      "               14,\n",
      "               14,\n",
      "               15,\n",
      "               16,\n",
      "               17,\n",
      "               18,\n",
      "               19,\n",
      "               20],\n",
      " 'max_features': [0.4,\n",
      "                  0.4091836734693878,\n",
      "                  0.41836734693877553,\n",
      "                  0.4275510204081633,\n",
      "                  0.43673469387755104,\n",
      "                  0.4459183673469388,\n",
      "                  0.45510204081632655,\n",
      "                  0.4642857142857143,\n",
      "                  0.47346938775510206,\n",
      "                  0.4826530612244898,\n",
      "                  0.49183673469387756,\n",
      "                  0.5010204081632653,\n",
      "                  0.5102040816326531,\n",
      "                  0.5193877551020408,\n",
      "                  0.5285714285714286,\n",
      "                  0.5377551020408163,\n",
      "                  0.5469387755102041,\n",
      "                  0.5561224489795918,\n",
      "                  0.5653061224489796,\n",
      "                  0.5744897959183674,\n",
      "                  0.5836734693877551,\n",
      "                  0.5928571428571429,\n",
      "                  0.6020408163265306,\n",
      "                  0.6112244897959184,\n",
      "                  0.6204081632653061,\n",
      "                  0.6295918367346939,\n",
      "                  0.6387755102040816,\n",
      "                  0.6479591836734694,\n",
      "                  0.6571428571428571,\n",
      "                  0.6663265306122449,\n",
      "                  0.6755102040816326,\n",
      "                  0.6846938775510204,\n",
      "                  0.6938775510204082,\n",
      "                  0.7030612244897959,\n",
      "                  0.7122448979591837,\n",
      "                  0.7214285714285714,\n",
      "                  0.7306122448979592,\n",
      "                  0.7397959183673469,\n",
      "                  0.7489795918367347,\n",
      "                  0.7581632653061224,\n",
      "                  0.7673469387755102,\n",
      "                  0.7765306122448979,\n",
      "                  0.7857142857142857,\n",
      "                  0.7948979591836735,\n",
      "                  0.8040816326530612,\n",
      "                  0.813265306122449,\n",
      "                  0.8224489795918367,\n",
      "                  0.8316326530612245,\n",
      "                  0.8408163265306122,\n",
      "                  0.85],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [10,\n",
      "                  14,\n",
      "                  19,\n",
      "                  24,\n",
      "                  29,\n",
      "                  34,\n",
      "                  39,\n",
      "                  44,\n",
      "                  49,\n",
      "                  54,\n",
      "                  58,\n",
      "                  63,\n",
      "                  68,\n",
      "                  73,\n",
      "                  78,\n",
      "                  83,\n",
      "                  88,\n",
      "                  93,\n",
      "                  98,\n",
      "                  103,\n",
      "                  107,\n",
      "                  112,\n",
      "                  117,\n",
      "                  122,\n",
      "                  127,\n",
      "                  132,\n",
      "                  137,\n",
      "                  142,\n",
      "                  147,\n",
      "                  152,\n",
      "                  156,\n",
      "                  161,\n",
      "                  166,\n",
      "                  171,\n",
      "                  176,\n",
      "                  181,\n",
      "                  186,\n",
      "                  191,\n",
      "                  196,\n",
      "                  201,\n",
      "                  205,\n",
      "                  210,\n",
      "                  215,\n",
      "                  220,\n",
      "                  225,\n",
      "                  230,\n",
      "                  235,\n",
      "                  240,\n",
      "                  245,\n",
      "                  250]}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 10, stop = 250, num = 50)]\n",
    "# Number of features to consider at every split\n",
    "max_features = [float(x) for x in np.linspace(start = 0.4, stop = 0.85, num = 50)]\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(4, 20, num = 20)]\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True]\n",
    "\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "\n",
    "pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9994952728440769"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's see what we get on the cross validation score without the random search\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "np.mean(cross_val_score(RandomForestClassifier(),\n",
    "                        X_train, y_train, cv=10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KFold Cross Validation score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#k-fold cross validation score\n",
    "#np.mean(cross_val_score(RandomForestClassifier(n_jobs=-1, n_estimators=1060, max_features='sqrt'),\n",
    "#                X_train, y_train, cv=KFold(n_splits=5, shuffle=True, random_state=42)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Repeated Stratified KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.mean(cross_val_score(RandomForestClassifier(),\n",
    "#                X_train, y_train,\n",
    "#                cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=10, random_state=42)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I need to subset the data so the process won't take so long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sample(df,n):\n",
    "    \"\"\" Gets a random sample of n rows from df, without replacement.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas data frame, that you wish to sample from.\n",
    "    n: The number of rows you wish to sample.\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    return value: A random sample of n rows of df.\n",
    "\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "\n",
    "    >>> get_sample(df, 2)\n",
    "       col1 col2\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    \"\"\"\n",
    "    idxs = sorted(np.random.permutation(len(df))[:n])\n",
    "    return df.iloc[idxs].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we now take samples of the whole dataset without replacing\n",
    "X_train_sample = get_sample(pd.DataFrame(X_train), 20000)\n",
    "y_train_sample = get_sample(pd.DataFrame(y_train), 20000)\n",
    "\n",
    "X_test_sample = get_sample(pd.DataFrame(X_test), 20000)\n",
    "y_test_sample = get_sample(pd.DataFrame(y_test), 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 16.7 µs\n",
      "Fitting 10 folds for each of 100 candidates, totalling 1000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:   11.5s\n",
      "[Parallel(n_jobs=-1)]: Done 130 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 333 tasks      | elapsed: 10.4min\n",
      "[Parallel(n_jobs=-1)]: Done 616 tasks      | elapsed: 19.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1000 out of 1000 | elapsed: 29.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ROC of the tuned classifier is 0.9282074793919686\n",
      "The F1 of the tuned classifier is 0.7200000000000001\n",
      "{'min_samples_split': 5, 'min_samples_leaf': 2, 'n_estimators': 58, 'bootstrap': True, 'max_features': 0.45510204081632655, 'max_depth': 20}\n"
     ]
    }
   ],
   "source": [
    "rm_clf = RandomForestClassifier(random_state=42)\n",
    "\n",
    "\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "grid = RandomizedSearchCV(rm_clf, param_distributions=random_grid,\n",
    "                              n_iter = 100, scoring='f1', \n",
    "                              cv =KFold(n_splits=10, shuffle=True, random_state=42), verbose=2, random_state=42, n_jobs=-1,\n",
    "                              return_train_score=True)\n",
    "\n",
    "# Fit GridSearchCV\n",
    "%time\n",
    "grid.fit(X_train_sample, y_train_sample)\n",
    "\n",
    "\n",
    "y_pred = grid.predict(X_test_sample)\n",
    "\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test_sample, y_pred)\n",
    "roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "f1 = f1_score(y_test_sample,y_pred)\n",
    "\n",
    "# Print results\n",
    "print(\"The ROC of the tuned classifier is\",roc_auc)\n",
    "print(\"The F1 of the tuned classifier is\",f1)\n",
    "\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 28s, sys: 64 ms, total: 1min 28s\n",
      "Wall time: 6.5 s\n",
      "['roc_auc-->', 0.9522155209357432, 'F1_score-->', 0.8444444444444444, 'roc_auc_score-->', 0.9522155209357432]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.7350427350427351]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.99993380004303]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.00155\n",
      "Accuracy  :  0.99845\n",
      "Precision  :  0.8431372549019608\n",
      "Recall  :  0.6515151515151515\n",
      "Specificity  :  0.6515151515151515\n",
      "False positive rate  :  0.0004013243704223939\n",
      "True positive rate  :  0.6515151515151515\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGTpJREFUeJzt3Xu0XHWV4PHvJunYEBgSIEZIGAbaqCADNMOzWxBFICDNo2W60RajEztOGmwdtQV8DCrq4PI1bUtjX5oMQXoIEVrIuAgQHgqOAmGGx5Dwuo0yJAoEAkhgDeFy9/xxT+IlklwS6t7auef7yTorVb/6VdWvIKv22vvs+p3ITCRJ6rYtur0ASZLAgCRJKsKAJEkqwYAkSSrBgCRJKsGAJEkqwYAkSSrBgCRJKsGAJEktFBFzIuLxiLhn0NgXImJ5RNzZHMcMeuzMiOiNiPsj4qhB49Obsd6IOGPQ+K4RcWszfmlEjBtyTcO9U8OLTzzkVhAaMVvudEi3l6CW6Vu9PDr1Wp38vvy9HXbb4Loi4lBgFXBRZu7ZjH0BWJWZ31hn7h7AJcABwE7AdcCbmocfAI4AlgGLgfdm5tKImA/8c2bOi4jvAXdl5nkbWpMZkiS1UGbeBKx8ldOPB+Zl5guZ+Qugl4HgdADQm5kPZeZqYB5wfEQE8E7gsub5c4EThnoTA5IkVdH/UueOTXdaRNzdlPQmNmNTgEcGzVnWjK1vfHvg6czsW2d8gwxIkjQKRcSsiLh90DHrVTztPOAPgH2AXwPfHNZFrmPsSL6ZJGkDsr9zL5XZA/Rs5HMeW3M7Is4HftTcXQ7sPGjq1GaM9Yw/CUyIiLFNljR4/nqZIUlSFf39nTs2QUTsOOjuicCaDrwFwMkR8bqI2BWYBtzGQBPDtKajbhxwMrAgB7rlbgROap4/A7hyqPc3Q5KkFoqIS4DDgB0iYhlwFnBYROwDJPBL4CMAmbmk6ZpbCvQBp2bmS83rnAZcA4wB5mTmkuYtTgfmRcSXgTuAC4Zck23fGk1s+9ZI62Tb9+pfLenY9+W4nd7asXWNFDMkSapiE0tto4XnkCRJJZghSVIVHeyy2xwZkCSpitf2g9bNniU7SVIJZkiSVIUlO0lSCXbZSZLUfWZIklREWrKTJJVgyU6SpO4zQ5KkKizZSZJK8IexkiR1nxmSJFVhyU6SVIJddpIkdZ8ZkiRVYclOklSCJTtJkrrPDEmSishs9++QDEiSVEXLzyFZspMklWCGJElVtLypwYAkSVW0vGRnQJKkKtxcVZKk7jNDkqQqLNlJkkpoeVODJTtJUglmSJJUhSU7SVIJluwkSeo+MyRJqqLlGZIBSZKKaPtu35bsJEklmCFJUhWW7CRJJbS87duSnSSpBDMkSarCkp0kqQRLdpIkdZ8ZkiRVYclOklSCJTtJkrrPDEmSqrBkJ0kqoeUByZKdJKkEMyRJqqLlTQ0GJEmqwpKdJEndZ4YkSVVYspMklWDJTpKk7jNDkqQqLNlJkkqwZCdJUveZIUlSFS3PkAxIklRFZrdX0FWW7CRJJZghSVIVluwkSSW0PCBZspMklWCGJElV+MNYSVIJluwkSeo+MyRJqsLfIUmSSujv79wxhIiYExGPR8Q9g8a+HhH3RcTdEfHDiJgw6LEzI6I3Iu6PiKMGjU9vxnoj4oxB47tGxK3N+KURMW6oNRmQJKmdLgSmrzO2CNgzM/cCHgDOBIiIPYCTgbc2z/n7iBgTEWOAc4GjgT2A9zZzAb4GfDsz3wg8BcwcakEGJEmqYgQzpMy8CVi5zti1mdnX3L0FmNrcPh6Yl5kvZOYvgF7ggObozcyHMnM1MA84PiICeCdwWfP8ucAJQ63Jc0iSVEWttu//AFza3J7CQIBaY1kzBvDIOuMHAtsDTw8KboPnr5cZkiSNQhExKyJuH3TM2ojnfhboA/5p+Fb4u8yQJKmI7O9cl11m9gA9G/u8iPggcCxweObatr/lwM6Dpk1txljP+JPAhIgY22RJg+evlxmSJFUxgueQXklETAc+DRyXmc8PemgBcHJEvC4idgWmAbcBi4FpTUfdOAYaHxY0gexG4KTm+TOAK4d6fwOSJLVQRFwC/Bx4c0Qsi4iZwHeBbYBFEXFnRHwPIDOXAPOBpcDVwKmZ+VKT/ZwGXAPcC8xv5gKcDnwiInoZOKd0wZBrymH+IdaLTzzU7l96aURtudMh3V6CWqZv9fLo1Gs9f95HO/Z9udXsv+vYukaK55AkqYoOnkPaHFmykySVYIYkSVW0fLdvA5IkVWFAkiSV4G7fkiR1nxmSJFVhyU7D6XNf/RY3/c/b2G7iBK64+HsAnHvBxVy+4GomTtgWgI99ZAaH/tEBPP3Mb/hPn/0K99z3ACccfQSf/eRfrX2dJfc9yOe+8i3+3wsvcMjB+3Pmx/8jEcEnP/9f+OX/XQbAs6tWsc3WW3P53HNH/oNqs9X7wC08u2oVL73UT19fHwcdfEy3l9ReLW/7NiANsxOOOYL3vec4PnP2N142fsqfn8CH3nfSy8bGjRvHR//yFB586GF6H3r4ZY+d/Y3v8oXT/5q93voWZn/qP/PTW27nkIP355tnn7l2ztf/7ny2Hr/V8H0YjVrvOuLf8+STT3V7GWq5Ic8hRcRbIuL0iPhOc5weEbuPxOJGg/32+bds+6+2eVVzt9ry99l37z153biXX1hxxRMree6559l7z92JCI6bfjg33Pzzl83JTK6+4SaOOeKwTi1d0kjL/s4dm6ENBqSIOJ2BCy4FAxvp3dbcvmTwpWq18S65/H9w4gdm87mvfotnfvPsBuc+tuIJJr9+h7X3J0/agcdWPPmyOf/rrnvYfuJEdtl5yEuOSC+TmSy86hJuvWUhH575F91eTrv1Z+eOzdBQGdJMYP/MPCczL26Ocxi4SuB6L0c7+Doc/3jRJZ1c76jw5ye+m4Xz53D5hecyafvt+Pp3z3/Nr3nVoh9zzBFv78Dq1DZvf8eJHHDgdI79k/cze/YHOeRtB3Z7SWqpoQJSP7DTK4zv2Dz2ijKzJzP3y8z9PvyB976W9Y1KO2w3kTFjxrDFFltw0nFHc8/SBzY4f/KkHXjs8SfW3n9sxRNMnrT92vt9fS9x3U9+xvTDDx22NWv0+tWvHgVgxYonufLKhey//z5dXlF7ZX9/x47N0VAB6ePA9RGxMCJ6muNq4HrgY8O/vNFpxRO/vYz99T/5GW/cbZcNzp+0w3aMH78Vd91zL5nJgquv5x1vO2jt47fcfge77TKVN7x+0rCtWaPTVlttydZbj197+4h3vZ0lS+7v8qparOUluw122WXm1RHxJgZKdGtOTiwHFmfmS8O9uNHgb846h8V33M3TT/+Gw094P3818xQW33E39z/4EARMecNkzvr0X6+df+R7ZrDqued5sa+PG27+GT3f/gp/sOsufO6Tp/627fug/Tnk4P3XPmfhdT/h6Hcd1oVPp83d5MmTuOwHA5epGTt2DPPmXcE11/64u4tSa3k9JI0qXg9JI62T10N67svv79j35fjPXez1kCRJm2gzLbV1invZSZJKMEOSpCo20+64TjEgSVIVluwkSeo+MyRJqmIz3YOuUwxIklSFJTtJkrrPDEmSithc96DrFAOSJFVhyU6SpO4zQ5KkKlqeIRmQJKmKlrd9W7KTJJVghiRJVViykyRVkC0PSJbsJEklmCFJUhUtz5AMSJJURct3arBkJ0kqwQxJkqqwZCdJKqHlAcmSnSSpBDMkSSois90ZkgFJkqqwZCdJUveZIUlSFS3PkAxIklSEe9lJklSAGZIkVdHyDMmAJElVtHsrO0t2kqQazJAkqYi2NzUYkCSpipYHJEt2kqQSzJAkqYqWNzUYkCSpiLafQ7JkJ0kqwQxJkqqwZCdJqsCSnSRJBZghSVIVluwkSRWkAUmSVELLA5LnkCRJJZghSVIRluwkSTW0PCBZspMklWCGJElFWLKTJJXQ9oBkyU6SVIIBSZKKyP7OHa9GRHwsIu6JiCUR8fFmbLuIWBQRDzZ/T2zGIyK+ExG9EXF3ROw76HVmNPMfjIgZm/r5DUiSVEVG544hRMSewF8CBwB7A8dGxBuBM4DrM3MacH1zH+BoYFpzzALOa15nO+As4MDmtc5aE8Q2lgFJktppd+DWzHw+M/uAnwB/ChwPzG3mzAVOaG4fD1yUA24BJkTEjsBRwKLMXJmZTwGLgOmbsiADkiQV0cmSXUTMiojbBx2z1nm7e4BDImL7iNgKOAbYGZicmb9u5jwKTG5uTwEeGfT8Zc3Y+sY3ml12klRE9g9danvVr5XZA/Rs4PF7I+JrwLXAc8CdwEvrzMmIGLGLNJkhSVJLZeYFmfnvMvNQ4CngAeCxphRH8/fjzfTlDGRQa0xtxtY3vtEMSJJURBe67F7f/P2vGTh/9N+BBcCaTrkZwJXN7QXAB5puu4OAZ5rS3jXAkRExsWlmOLIZ22iW7CSpiHwV3XEddnlEbA+8CJyamU9HxDnA/IiYCTwM/Fkz9yoGzjP1As8DHxpYc66MiLOBxc28L2Xmyk1ZjAFJkloqMw95hbEngcNfYTyBU9fzOnOAOa91PQYkSSqi7VsHGZAkqYhOdtltjmxqkCSVYIYkSUXkiP3ipyYDkiQVYclOkqQCzJAkqYi2Z0gGJEkqou3nkCzZSZJKMEOSpCIs2UmSSujCXnalWLKTJJVghiRJRbiXnSSphH5LdpIkdZ8ZkiQV0famBgOSJBXR9rZvS3aSpBLMkCSpiLZvHWRAkqQiLNlJklSAGZIkFdH23yEZkCSpiLa3fVuykySVYIYkSUXYZSdJKqHt55As2UmSSjBDkqQi2t7UYECSpCLafg7Jkp0kqYRhz5C23OmQ4X4LSRoV2t7UYMlOkopo+zkkS3aSpBLMkCSpCEt2kqQSWt5kZ0CSpCraniF5DkmSVIIZkiQV0fYuOwOSJBXR3+0FdJklO0lSCWZIklREYslOklRAf8v7vi3ZSZJKMEOSpCL6LdlJkipo+zkkS3aSpBLMkCSpiLb/DsmAJElFWLKTJKkAMyRJKsKSnSSphLYHJEt2kqQSzJAkqYi2NzUYkCSpiP52xyNLdpKkGsyQJKkI97KTJJXQ8qtPWLKTJNVghiRJRbT9d0gGJEkqoj/afQ7Jkp0kqQQzJEkqou1NDQYkSSqi7eeQLNlJkkowQ5KkItq+dZABSZKKaPtODZbsJKmlImJCRFwWEfdFxL0RcXBEbBcRiyLiwebvic3ciIjvRERvRNwdEfsOep0ZzfwHI2LGpq7HgCRJRWQHj1fpb4GrM/MtwN7AvcAZwPWZOQ24vrkPcDQwrTlmAecBRMR2wFnAgcABwFlrgtjGMiBJUhH90bljKBGxLXAocAFAZq7OzKeB44G5zbS5wAnN7eOBi3LALcCEiNgROApYlJkrM/MpYBEwfVM+vwFJktppV2AF8N8i4o6I+MeIGA9MzsxfN3MeBSY3t6cAjwx6/rJmbH3jG82AJElF9HfwiIhZEXH7oGPWOm83FtgXOC8z/xB4jt+W5wDIzI2sAL42dtlJUhGd/ObPzB6gZwNTlgHLMvPW5v5lDASkxyJix8z8dVOSe7x5fDmw86DnT23GlgOHrTP+401ZsxmSJLVQZj4KPBIRb26GDgeWAguANZ1yM4Arm9sLgA803XYHAc80pb1rgCMjYmLTzHBkM7bRzJAkqYgu/DD2o8A/RcQ44CHgQwwkKvMjYibwMPBnzdyrgGOAXuD5Zi6ZuTIizgYWN/O+lJkrN2UxBiRJKmKk97LLzDuB/V7hocNfYW4Cp67ndeYAc17reizZSZJKMEOSpCLavtu3AUmSish2b2VnyU6SVIMZkiQVYclOklRC2wOSJTtJUglmSJJUxIhtGleUAUmSimj7Jcwt2UmSSjBDkqQi2t7UYECSpCLaHpAs2UmSSjBDkqQi7LKTJJXQ9i47A5IkFeE5JEmSCjBDkqQiPIckSSqhv+UhyZKdJKkEMyRJKqLtTQ0GJEkqot0FO0t2kqQizJAkqQhLdpKkEtq+U4MlO0lSCWZIklRE23+HZECSpCLaHY4s2UmSijBDkqQi7LKTJJXQ9nNIluwkSSWYIUlSEe3OjwxIklRG288hWbKTJJVghiRJRbS9qcGAJElFtDscWbKTJBVhhiRJRbS9qcGAJElFZMuLdpbsJEklmCFJUhGW7CRJJbS97duSnSSpBDMkSSqi3fmRAUmSyrBkp1KmTt2J6679AXffdSN33XkDHz1tZreXpFHm/J5v8qtld3HnHdevHdtrrz346U0LuON/X8cVP7yQbbbZuosrVFsZkIrp6+vjbz79Rfba+x388dv+hNmzP8juu0/r9rI0ilx00XzefexfvGzsH773dT7z2a/yh/u+iyuuWMinPjm7S6trt/4OHpsjA1Ixjz76OHfceQ8Aq1Y9x333PciUnd7Q5VVpNLn5p7ey8qmnXzb2pmm7cdPNtwBw3fU3c+KJx3Rjaa2XHfyzOdrkgBQRH+rkQvS7dtllKvvsvSe33nZHt5eiUW7p0gc47rijADjpPcey89SdurwitdFryZC+2LFV6HeMH78V8y89n0986iyefXZVt5ejUe7Dsz7B7I/M4NZbFrLNNuNZvfrFbi+pldpesttgl11E3L2+h4DJG3jeLGAWQIzZli22GL/JC2yjsWPH8oNLz+eSS37IFVcs7PZy1AL33/8vHP3u9wEwbdpuHHP04V1eUTttrqW2Thmq7XsycBTw1DrjAfxsfU/KzB6gB2DsuCnt/i+8Cc7v+Sb33tfLf/3bnm4vRS0xadL2rFjxJBHBZ878GP/Q8/1uL0ktNFRA+hGwdWbeue4DEfHjYVlRy/3xH+3PKe8/ibv/z1JuX3wtAJ///DksvPqGLq9Mo8XF3z+Xtx96MDvssB2/fOh2vvilb7D11uOZPfuDAFxxxVVcOPfS7i6ypTbXUlunRObwJjBmSJJGs77Vy6NTr3XKLn/ase/L7z/8zx1b10ix7VuSVIJbB0lSEW0vJxmQJKkI97KTJKkAMyRJKsLfIUmSSmh727clO0lSCWZIklRE25saDEiSVETbzyFZspMklWCGJElFtL2pwYAkSUUM996i1Vmyk6QWiojfj4jbIuKuiFgSEV9sxneNiFsjojciLo2Icc3465r7vc3j/2bQa53ZjN8fEUdt6poMSJJURD/ZseNVeAF4Z2buDewDTI+Ig4CvAd/OzDcycC28mc38mcBTzfi3m3lExB7AycBbgenA30fEmE35/AYkSSpiJC9hngNWNXd/rzkSeCdwWTM+FzihuX18c5/m8cMjIprxeZn5Qmb+AugFDtjYzw4GJEkqIzv4JyJmRcTtg45Z675fRIyJiDuBx4FFwL8AT2dmXzNlGTCluT0FeASgefwZYPvB46/wnI1iU4MkjUKZ2QP0DDHnJWCfiJgA/BB4y0isbX0MSJJURLd2asjMpyPiRuBgYEJEjG2yoKnA8mbacmBnYFlEjAW2BZ4cNL7G4OdsFEt2klREZnbsGEpETGoyIyJiS+AI4F7gRuCkZtoM4Mrm9oLmPs3jN+TAGy0ATm668HYFpgG3bcrnN0OSpHbaEZjbdMRtAczPzB9FxFJgXkR8GbgDuKCZfwHw/YjoBVYy0FlHZi6JiPnAUqAPOLUpBW60GO4fYo0dN6Xdv/SSNKr1rV4enXqto3Y+umPfl9c8srBj6xopZkiSVISbq0qSVIAZkiQV4fWQJEkluLmqJEkFmCFJUhGW7CRJJdhlJ0lSAWZIklREf8ubGgxIklREu8ORJTtJUhFmSJJUhF12kqQS2h6QLNlJkkowQ5KkItq+dZABSZKKsGQnSVIBZkiSVETbtw4yIElSEW0/h2TJTpJUghmSJBXR9qYGA5IkFWHJTpKkAsyQJKkIS3aSpBLa3vZtyU6SVIIZkiQV4RVjJUklWLKTJKkAMyRJKsKSnSSpBEt2kiQVYIYkSUVYspMklWDJTpKkAsyQJKkIS3aSpBIs2UmSVIAZkiQVkdnf7SV0lQFJkopo+/WQLNlJkkowQ5KkItIuO0lSBZbsJEkqwAxJkoqwZCdJKqHtOzVYspMklWCGJElFtH3rIAOSJBXhOSRJUgm2fUuSVIAZkiQVYclOklSCbd+SJBVghiRJRViykySVYJedJEkFmCFJUhGW7CRJJdhlJ0lSAWZIklSEm6tKkkqwZCdJUgFmSJJUhF12kqQS2n4OyZKdJKkEA5IkFZGZHTtejYiYHhH3R0RvRJwxzB9vSJbsJKmIkTyHFBFjgHOBI4BlwOKIWJCZS0dsEeswQ5KkdjoA6M3MhzJzNTAPOL6bCzIgSVIR2cHjVZgCPDLo/rJmrGuGvWTXt3p5DPd7jEYRMSsze7q9DrWH/+a6r5PflxExC5g1aKin+v9fM6S6Zg09Reoo/82NIpnZk5n7DTrWDUbLgZ0H3Z/ajHWNAUmS2mkxMC0ido2IccDJwIJuLsguO0lqoczsi4jTgGuAMcCczFzSzTUZkOoqXevVqOS/uZbJzKuAq7q9jjWi7XsnSZJq8BySJKkEA1JB1bbz0OgVEXMi4vGIuKfba5EMSMUM2s7jaGAP4L0RsUd3V6VR7EJgercXIYEBqaJy23lo9MrMm4CV3V6HBAakispt5yFJI8GAJEkqwYBUT7ntPCRpJBiQ6im3nYckjQQDUjGZ2Qes2c7jXmB+t7fz0OgVEZcAPwfeHBHLImJmt9ek9nKnBklSCWZIkqQSDEiSpBIMSJKkEgxIkqQSDEiSpBIMSJKkEgxIkqQSDEiSpBL+P/GnNwIuAWQyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml_rfeX = RandomForestClassifier(n_jobs=-1, n_estimators=58, min_samples_leaf=2,min_samples_split=5, max_features=0.45510204081632655, bootstrap=True, max_depth=20)\n",
    "%time ml_rfeX.fit(X_train, y_train)\n",
    "pred_d = ml_rfeX.predict(X_test)\n",
    "print_score(pred_d,ml_rfeX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Third Result using 16 cores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.9 s, sys: 44 ms, total: 59 s\n",
      "Wall time: 4.48 s\n",
      "['roc_auc-->', 0.9520170032014923, 'F1_score-->', 0.7450980392156863, 'roc_auc_score-->', 0.9520170032014923]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.5765765765765766]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.9996525594785084]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.00235\n",
      "Accuracy  :  0.99765\n",
      "Precision  :  0.7111111111111111\n",
      "Recall  :  0.48484848484848486\n",
      "Specificity  :  0.48484848484848486\n",
      "False positive rate  :  0.0006521521019363901\n",
      "True positive rate  :  0.48484848484848486\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGOxJREFUeJzt3X20nWV55/HvBWksBEYISVNIkIKkIlKgVKi0hVqoEF5KoDAWqxhtapxMsO1yVMCXhSgwuBSpHRF7KBkCdIAUKmRcIEJAwVEgrEnCkISXU6wlEcJLCAqsEsK55o/zhB4CyUkOO2dfOc/3k/Ws7H3vZ+99H8g617p+z73vHZmJJEndtk23JyBJEliQJElFWJAkSSVYkCRJJViQJEklWJAkSSVYkCRJJViQJEklWJAkqYUiYnZEPBkRDwwY+2JErIiIRc1x7IDHzoqI3oh4KCKOHjA+pRnrjYgzB4zvGRH3NOPXRsToQee0pXdqePnpR90KQsNmu90O6/YU1DJr16yITr1WJ39f/sq4vTY6r4g4HHgeuCIz92vGvgg8n5lfW+/cfYGrgUOA3YDbgN9sHn4YeB+wHFgAfCAzl0bEXOCfM/OaiPg2sDgzL9nYnOyQJKmFMvNOYNUmnj4VuCYzX8rMnwK99BenQ4DezHw0M9cA1wBTIyKAI4DrmufPAU4c7E0sSJJURd8rnTuG7vSIuL+J9HZuxiYCjw04Z3kztqHxXYDVmbl2vfGNsiBJ0ggUETMi4r4Bx4xNeNolwNuBA4HHgQu36CTXM2o430yStBHZ17mXyuwBejbzOSvX3Y6IS4HvNndXALsPOHVSM8YGxp8BdoqIUU2XNPD8DbJDkqQq+vo6dwxBROw64O5JwLoVePOAUyPiLRGxJzAZuJf+RQyTmxV1o4FTgXnZv1ruDuCU5vnTgBsHe387JElqoYi4GngvMC4ilgNnA++NiAOBBP4V+DhAZi5pVs0tBdYCszLzleZ1TgduAbYFZmfmkuYtzgCuiYhzgYXAZYPOyWXfGklc9q3h1sll32t+vqRjvy9H7/aujs1ruNghSVIVQ4zaRgqvIUmSSrBDkqQqOrjKbmtkQZKkKt7cB1q3ekZ2kqQS7JAkqQojO0lSCa6ykySp++yQJKmINLKTJJVgZCdJUvfZIUlSFUZ2kqQS/GCsJEndZ4ckSVUY2UmSSnCVnSRJ3WeHJElVGNlJkkowspMkqfvskCSpiMx2fw7JgiRJVbT8GpKRnSSpBDskSaqi5YsaLEiSVEXLIzsLkiRV4eaqkiR1nx2SJFVhZCdJKqHlixqM7CRJJdghSVIVRnaSpBKM7CRJ6j47JEmqouUdkgVJkopo+27fRnaSpBLskCSpCiM7SVIJLV/2bWQnSSrBDkmSqjCykySVYGQnSVL32SFJUhVGdpKkEozsJEnqPjskSarCyE6SVELLC5KRnSSpBDskSaqi5YsaLEiSVIWRnSRJ3WeHJElVGNlJkkowspMkqfvskCSpCiM7SVIJRnaSJHWfHZIkVdHyDsmCJElVZHZ7Bl1lZCdJKsEOSZKqMLKTJJXQ8oJkZCdJKsEOSZKq8IOxkqQSjOwkSeo+OyRJqsLPIUmSSujr69wxiIiYHRFPRsQDA8a+GhEPRsT9EfGdiNhpwGNnRURvRDwUEUcPGJ/SjPVGxJkDxveMiHua8WsjYvRgc7IgSVI7XQ5MWW/sVmC/zNwfeBg4CyAi9gVOBd7VPOdbEbFtRGwLXAwcA+wLfKA5F+ArwEWZuTfwLDB9sAlZkCSpimHskDLzTmDVemPfz8y1zd27gUnN7anANZn5Umb+FOgFDmmO3sx8NDPXANcAUyMigCOA65rnzwFOHGxOFiRJqiL7OnZExIyIuG/AMWMzZ/MXwM3N7YnAYwMeW96MbWh8F2D1gOK2bnyjXNQgSSNQZvYAPUN5bkR8DlgL/GNHJzUIC5IkFZF93V9lFxEfAY4Hjsx8ddnfCmD3AadNasbYwPgzwE4RMarpkgaev0FGdpJUxTBeQ3ojETEF+AxwQma+OOChecCpEfGWiNgTmAzcCywAJjcr6kbTv/BhXlPI7gBOaZ4/DbhxsPe3IElSC0XE1cBPgHdExPKImA58E9gRuDUiFkXEtwEycwkwF1gKfA+YlZmvNN3P6cAtwDJgbnMuwBnAJyOil/5rSpcNOqfcwh/EevnpR7vfg6o1ttvtsG5PQS2zds2K6NRrvXjJJzr2+3L7mf+jY/MaLl5DkqQqClxD6iYjO0lSCXZIklRFy3f7tiBJUhUWJElSCe72LUlS99khSVIVLY/s7JC2sM+f/3UOP+5UTvzQf3l17OLLruKIqR/i5GmzOHnaLO788b0ArH7uF3z09DM4+I9P4rwLv/Wa11ny4COcdNpMjnn/X3D+RZew7vNjt9x+F1M/+HF+6w+O5YFlDw/fD6at1qU9F/Lz5YtZtHD+q2Mnn3w8ixfdzpp/f4zfOWj/Ls6u5fqyc8dWyIK0hZ147Pv49tfPfd34aX92ItfPuZjr51zM4b93CACjR4/mEx87jU/N+svXnf/lr32TL57xV9x07WX82/Kf86O77wNg77324G/P/wK/c+B+W/YH0YhxxRVzOe74D75mbMmSB/nP7/8Yd911d5dmJW1CZBcR+9D/XRjrtg5fQf9eRcu25MRGincf+FuseHzlJp27/Xa/ykEH7Me/LX/8NeNPPb2KF154kQP2eycAJ0w5ktvv+gmHHXowb/+Nt3V8zhrZ7vrRPeyxx6TXjD34YG+XZqPXSCO7DYqIM+j/wqWgfyO9e5vbVw/8qlptvquv/9+c9OGZfP78r/PcL3650XNXPvU0E35t3Kv3J4wfx8qnntnSU5Q03IzsNmo6cHBmXpCZVzXHBfR/S+AGv4524BdD/cMVV3dyviPCn510HDfPnc31l1/M+F3G8tVvXtrtKUlS1w0W2fUBuwE/W2981+axNzTwi6HcXPX1xo3d+dXbp5xwDLM+ffZGz58wfhwrn3z61fsrn3qaCeN32WLzk9Qd2fJVdoMVpL8B5kfEI/zH19S+Ddib/i3HNQRPPb2K8ePGAjD/hz9m77322Oj548eNZcyY7Vn8wDL2f9c+zPvefP785D8ZjqlKGk5badTWKYN+/UREbEN/RDdwUcOCzHxlU96g7R3Sp8++gAUL72f16l+wy9id+K/TT2PBwvt56JFHIWDir0/g7M/81asF6qiTp/H8Cy/y8tq1/KcdxtBz0Xm8fc89eGDZw3z+vK/z7y+9xGHvOZjPfnImEcFtP/w//PeLLmHV6ufYcYcd2GfyXvRcdF6Xf+ru8esnBnfVlRfzh4cfyrhxY1m58mnO+dLXWPXsar5x0bmMHz+W1at/weLFSzh2vZV4emOd/PqJF877cMd+X4753BVb3ddP+H1IGlEsSBpuHS1I536ocwXp81dtdQXJnRokqYqWR3Z+MFaSVIIdkiRV4So7SVIJRnaSJHWfHZIkVdHyvewsSJJUhZGdJEndZ4ckSUW4l50kqQYjO0mSus8OSZKqaHmHZEGSpCpavuzbyE6SVIIdkiRVYWQnSaogW16QjOwkSSXYIUlSFS3vkCxIklRFy3dqMLKTJJVghyRJVRjZSZJKaHlBMrKTJJVghyRJRWS2u0OyIElSFUZ2kiR1nx2SJFXR8g7JgiRJRbiXnSRJBdghSVIVLe+QLEiSVEW7t7IzspMk1WCHJElFtH1RgwVJkqpoeUEyspMklWCHJElVtHxRgwVJkopo+zUkIztJUgl2SJJUhZGdJKkCIztJkgqwQ5KkKozsJEkVpAVJklRCywuS15AkSSXYIUlSEUZ2kqQaWl6QjOwkSSXYIUlSEUZ2kqQS2l6QjOwkqaUi4q8j4oGIWBIRf9OMjY2IWyPikebvnZvxiIi/i4jeiLg/Ig4a8DrTmvMfiYhpQ52PBUmSisi+zh2DiYj9gI8BhwAHAMdHxN7AmcD8zJwMzG/uAxwDTG6OGcAlzeuMBc4Gfrd5rbPXFbHNZUGSpCoyOncM7p3APZn5YmauBX4I/CkwFZjTnDMHOLG5PRW4IvvdDewUEbsCRwO3ZuaqzHwWuBWYMpQf34IkSe30AHBYROwSEdsDxwK7AxMy8/HmnCeACc3ticBjA56/vBnb0Phmc1GDJBXRyUUNETGD/mhtnZ7M7Hn1vTKXRcRXgO8DLwCLgFdeM5/MjIhh+04MC5IkFZF9mxS1bdpr9RefnkHOuQy4DCAizqe/u1kZEbtm5uNNJPdkc/oK+juodSY1YyuA9643/oOhzNnITpJaKiJ+rfn7bfRfP/pfwDxg3Uq5acCNze15wIeb1XbvAZ5ror1bgKMiYudmMcNRzdhms0OSpCK68Dmk6yNiF+BlYFZmro6IC4C5ETEd+Bnw/ubcm+i/ztQLvAh8FCAzV0XEl4EFzXlfysxVQ5mMBUmSishNWx3XwffLw95g7BngyDcYT2DWBl5nNjD7zc7HyE6SVIIdkiQV0fatgyxIklREJ1fZbY2M7CRJJdghSVIROWwfQa3JgiRJRRjZSZJUgB2SJBXR9g7JgiRJRbT9GpKRnSSpBDskSSrCyE6SVMJw72VXjZGdJKkEOyRJKsK97CRJJfQZ2UmS1H12SJJURNsXNViQJKmIti/7NrKTJJVghyRJRbR96yALkiQVYWQnSVIBdkiSVETbP4dkQZKkItq+7NvITpJUgh2SJBXhKjtJUgltv4ZkZCdJKsEOSZKKaPuiBguSJBXR9mtIRnaSpBK2eIe03W6Hbem3kKQRoe2LGozsJKmItl9DMrKTJJVghyRJRRjZSZJKaPkiOwuSJFXR9g7Ja0iSpBLskCSpiLavsrMgSVIRfd2eQJcZ2UmSSrBDkqQiEiM7SVIBfS1f921kJ0kqwQ5JkoroM7KTJFXQ9mtIRnaSpBLskCSpiLZ/DsmCJElFGNlJklSAHZIkFWFkJ0kqoe0FychOklSCHZIkFdH2RQ0WJEkqoq/d9cjITpJUgx2SJBXhXnaSpBJa/u0TRnaSpBrskCSpiLZ/DsmCJElF9EW7ryEZ2UmSSrBDkqQi2r6owYIkSUW0/RqSkZ0kqQQ7JEkqou1bB1mQJKmItu/UYGQnSS0VETtFxHUR8WBELIuIQyNibETcGhGPNH/v3JwbEfF3EdEbEfdHxEEDXmdac/4jETFtqPOxIElSEdnBYxN9A/heZu4DHAAsA84E5mfmZGB+cx/gGGByc8wALgGIiLHA2cDvAocAZ68rYpvLgiRJRfRF547BRMRbgcOBywAyc01mrgamAnOa0+YAJza3pwJXZL+7gZ0iYlfgaODWzFyVmc8CtwJThvLzW5AkqZ32BJ4C/mdELIyIf4iIMcCEzHy8OecJYEJzeyLw2IDnL2/GNjS+2SxIklREXwePiJgREfcNOGas93ajgIOASzLzt4EX+I94DoDM3MwE8M1xlZ0kFdHJ3/yZ2QP0bOSU5cDyzLynuX8d/QVpZUTsmpmPN5Hck83jK4DdBzx/UjO2AnjveuM/GMqc7ZAkqYUy8wngsYh4RzN0JLAUmAesWyk3DbixuT0P+HCz2u49wHNNtHcLcFRE7NwsZjiqGdtsdkiSVEQXPhj7CeAfI2I08CjwUfoblbkRMR34GfD+5tybgGOBXuDF5lwyc1VEfBlY0Jz3pcxcNZTJWJAkqYjh3ssuMxcB736Dh458g3MTmLWB15kNzH6z8zGykySVYIckSUW0fbdvC5IkFZHt3srOyE6SVIMdkiQVYWQnSSqh7QXJyE6SVIIdkiQVMWybxhVlQZKkItr+FeZGdpKkEuyQJKmIti9qsCBJUhFtL0hGdpKkEuyQJKkIV9lJkkpo+yo7C5IkFeE1JEmSCrBDkqQivIYkSSqhr+UlychOklSCHZIkFdH2RQ0WJEkqot2BnZGdJKkIOyRJKsLITpJUQtt3ajCykySVYIckSUW0/XNIFiRJKqLd5cjITpJUhB2SJBXhKjtJUgltv4ZkZCdJKsEOSZKKaHd/ZEGSpDLafg3JyE6SVIIdkiQV0fZFDRYkSSqi3eXIyE6SVIQdkiQV0fZFDRYkSSoiWx7aGdlJkkqwQ5KkIozsJEkltH3Zt5GdJKkEOyRJKqLd/ZEFSZLKMLJTKZMm7cZt3/8n7l98B4sX3c4nTp/e7SlphLm050J+vnwxixbOf3Vs//335Ud3zmPh/72NG75zOTvuuEMXZ6i2siAVs3btWj79mXPY/4A/4vf/4E+YOfMjvPOdk7s9LY0gV1wxl+OO/+Brxv7+21/ls587n98+6I+54Yab+dR/m9ml2bVbXwePrZEFqZgnnniShYseAOD551/gwQcfYeJuv97lWWkkuetH97Dq2dWvGfvNyXtx5113A3Db/Ls46aRjuzG11ssO/tkaDbkgRcRHOzkRvd4ee0ziwAP24557F3Z7Khrhli59mBNOOBqAU04+nt0n7dblGamN3kyHdE7HZqHXGTNme+Zeeymf/NTZ/PKXz3d7Ohrh/nLGJ5n58Wncc/fN7LjjGNasebnbU2qltkd2G11lFxH3b+ghYMJGnjcDmAEQ276VbbYZM+QJttGoUaP4p2sv5eqrv8MNN9zc7emoBR566F845rg/B2Dy5L049pgjuzyjdtpao7ZOGWzZ9wTgaODZ9cYD+PGGnpSZPUAPwKjRE9v9X3gILu25kGUP9vK33+jp9lTUEuPH78JTTz1DRPDZs/6av++5sttTUgsNVpC+C+yQmYvWfyAifrBFZtRyv/97B3Pah07h/v+3lPsWfB+AL3zhAm7+3u1dnplGiquuvJg/PPxQxo0by78+eh/nfOlr7LDDGGbO/AgAN9xwE5fPuba7k2yprTVq65TI3LINjB2SpJFs7ZoV0anXOm2PP+3Y78srf/bPHZvXcHHZtySpBLcOkqQi2h4nWZAkqQj3spMkqQA7JEkqws8hSZJKaPuybyM7SVIJdkiSVETbFzVYkCSpiLZfQzKykySVYIckSUW0fVGDBUmSitjSe4tWZ2QnSS0UEb8aEfdGxOKIWBIR5zTje0bEPRHRGxHXRsToZvwtzf3e5vHfGPBaZzXjD0XE0UOdkwVJkoroIzt2bIKXgCMy8wDgQGBKRLwH+ApwUWbuTf934U1vzp8OPNuMX9ScR0TsC5wKvAuYAnwrIrYdys9vQZKkIobzK8yz3/PN3V9pjgSOAK5rxucAJza3pzb3aR4/MiKiGb8mM1/KzJ8CvcAhm/uzgwVJksrIDv6JiBkRcd+AY8b67xcR20bEIuBJ4FbgX4DVmbm2OWU5MLG5PRF4DKB5/Dlgl4Hjb/CczeKiBkkagTKzB+gZ5JxXgAMjYifgO8A+wzG3DbEgSVIR3dqpITNXR8QdwKHAThExqumCJgErmtNWALsDyyNiFPBW4JkB4+sMfM5mMbKTpCIys2PHYCJifNMZERHbAe8DlgF3AKc0p00Dbmxuz2vu0zx+e/a/0Tzg1GYV3p7AZODeofz8dkiS1E67AnOaFXHbAHMz87sRsRS4JiLOBRYClzXnXwZcGRG9wCr6V9aRmUsiYi6wFFgLzGqiwM0WW/qDWKNGT2z3J70kjWhr16yITr3W0bsf07Hfl7c8dnPH5jVc7JAkqQg3V5UkqQA7JEkqwu9DkiSV4OaqkiQVYIckSUUY2UmSSnCVnSRJBdghSVIRfS1f1GBBkqQi2l2OjOwkSUXYIUlSEa6ykySV0PaCZGQnSSrBDkmSimj71kEWJEkqwshOkqQC7JAkqYi2bx1kQZKkItp+DcnITpJUgh2SJBXR9kUNFiRJKsLITpKkAuyQJKkIIztJUgltX/ZtZCdJKsEOSZKK8BtjJUklGNlJklSAHZIkFWFkJ0kqwchOkqQC7JAkqQgjO0lSCUZ2kiQVYIckSUUY2UmSSjCykySpADskSSois6/bU+gqC5IkFdH270MyspMklWCHJElFpKvsJEkVGNlJklSAHZIkFWFkJ0kqoe07NRjZSZJKsEOSpCLavnWQBUmSivAakiSpBJd9S5JUgB2SJBVhZCdJKsFl35IkFWCHJElFGNlJkkpwlZ0kSQXYIUlSEUZ2kqQSXGUnSVIBdkiSVISbq0qSSjCykySpADskSSrCVXaSpBLafg3JyE6SVIIFSZKKyMyOHZsiIqZExEMR0RsRZ27hH29QRnaSVMRwXkOKiG2Bi4H3AcuBBRExLzOXDtsk1mOHJEntdAjQm5mPZuYa4BpgajcnZEGSpCKyg8cmmAg8NuD+8masa7Z4ZLd2zYrY0u8xEkXEjMzs6fY81B7+m+u+Tv6+jIgZwIwBQz3V///aIdU1Y/BTpI7y39wIkpk9mfnuAcf6xWgFsPuA+5Oasa6xIElSOy0AJkfEnhExGjgVmNfNCbnKTpJaKDPXRsTpwC3AtsDszFzSzTlZkOoqnfVqRPLfXMtk5k3ATd2exzrR9r2TJEk1eA1JklSCBamgatt5aOSKiNkR8WREPNDtuUgWpGIGbOdxDLAv8IGI2Le7s9IIdjkwpduTkMCCVFG57Tw0cmXmncCqbs9DAgtSReW285Ck4WBBkiSVYEGqp9x2HpI0HCxI9ZTbzkOShoMFqZjMXAus285jGTC329t5aOSKiKuBnwDviIjlETG923NSe7lTgySpBDskSVIJFiRJUgkWJElSCRYkSVIJFiRJUgkWJElSCRYkSVIJFiRJUgn/H/f4rzhhIaR+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml_rfe = RandomForestClassifier(n_jobs=-1, n_estimators=49, min_samples_leaf=2, max_features=0.45510204081632655, bootstrap=True, max_depth=10)\n",
    "%time ml_rfe.fit(X_train, y_train)\n",
    "pred_d = ml_rfe.predict(X_test)\n",
    "print_score(pred_d,ml_rfe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Second Results from the random Search\n",
    "\n",
    "We still have a highly complex model but we are becoming better, I need to make the the model less complex on the training set so that we can generalize more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 15s, sys: 44 ms, total: 1min 15s\n",
      "Wall time: 5.39 s\n",
      "['roc_auc-->', 0.9757603544329325, 'F1_score-->', 0.7407407407407407, 'roc_auc_score-->', 0.9757603544329325]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.5671641791044776]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.999561610296368]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.0029\n",
      "Accuracy  :  0.9971\n",
      "Precision  :  0.5588235294117647\n",
      "Recall  :  0.5757575757575758\n",
      "Specificity  :  0.5757575757575758\n",
      "False positive rate  :  0.0015049663890839772\n",
      "True positive rate  :  0.5757575757575758\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGepJREFUeJzt3XvY1WWZ6PHvLUSZNYIHUMAcVDQ1R8dRK0szSQFN0RkrvUrJmE1bsV22dx7GZpOHq0vHLKeJdL8mE2gDkmii2xOheeigeI2HLaL5hgdeFFERjxXh++w/3h/OioQXcPGum/f3/XD9LtZ61rPWetYl17q979+9nl+UUpAkqdU2afUCJEkCA5IkKQkDkiQpBQOSJCkFA5IkKQUDkiQpBQOSJCkFA5IkKQUDkiTVUERMjoglEfFww9i3ImJRRDxQHYc1PHZmRLRHxGMRMbJhfFQ11h4RZzSMD4uIe6rxqyKiX7dr2tA7NfzphQVuBaEes+ngA1q9BNXMiuWLolmv1czvy3dttcMa1xURBwKvAVNLKR+qxr4FvFZK+c4qc3cDpgH7AYOBnwM7Vw//FjgE6ADmAseVUh6JiBnANaWU6RFxKfBgKeWSNa3JDEmSaqiUciewdC2njwGml1L+WEp5AminKzjtB7SXUhaUUpYD04ExERHAwcDV1fOnAEd19yYGJEnKovPN5h3r75SIeKgq6Q2oxoYACxvmdFRjqxvfElhWSlmxyvgaGZAkqReKiPERcV/DMX4tnnYJsCOwF/AscNEGXeQq+vbkm0mS1qB0Nu+lSmkD2tbxOc+tvB0RlwE3VHcXAds1TB1ajbGa8ReB/hHRt8qSGuevlhmSJGXR2dm8Yz1ExLYNd48GVnbgzQKOjYh3R8QwYDhwL11NDMOrjrp+wLHArNLVLXc7cEz1/LHAdd29vxmSJNVQREwDDgK2iogOYCJwUETsBRTgSeDLAKWUeVXX3CPACmBCKeXN6nVOAW4B+gCTSynzqrc4HZgeEecB9wOXd7sm277Vm9j2rZ7WzLbv5c/Ma9r3Zb/BuzdtXT3FDEmSsljPUltv4TkkSVIKZkiSlEUTu+w2RgYkScrinf2gdaNnyU6SlIIZkiRlYclOkpSCXXaSJLWeGZIkJVEs2UmSUrBkJ0lS65khSVIWluwkSSn4w1hJklrPDEmSsrBkJ0lKwS47SZJazwxJkrKwZCdJSsGSnSRJrWeGJElJlFLv3yEZkCQpi5qfQ7JkJ0lKwQxJkrKoeVODAUmSsqh5yc6AJElZuLmqJEmtZ4YkSVlYspMkpVDzpgZLdpKkFMyQJCkLS3aSpBQs2UmS1HpmSJKURc0zJAOSJCVR992+LdlJklIwQ5KkLCzZSZJSqHnbtyU7SVIKZkiSlIUlO0lSCpbsJElqPTMkScrCkp0kKQVLdpIktZ4ZkiRlYclOkpRCzQOSJTtJUgpmSJKURc2bGgxIkpSFJTtJklrPDEmSsrBkJ0lKwZKdJEmtZ4YkSVlYspMkpWDJTpKk1jNDkqQsap4hGZAkKYtSWr2ClrJkJ0lKwQxJkrKwZCdJSqHmAcmSnSQpBTMkScrCH8ZKklKwZCdJUuuZIUlSFv4OSZKUQmdn845uRMTkiFgSEQ83jF0YEY9GxEMRcW1E9G947MyIaI+IxyJiZMP4qGqsPSLOaBgfFhH3VONXRUS/7tZkQJKkevoxMGqVsdnAh0opfwP8FjgTICJ2A44Fdq+e88OI6BMRfYBJwGhgN+C4ai7ABcD3Sik7AS8B47pbkAFJkrLowQyplHInsHSVsVtLKSuqu78Bhla3xwDTSyl/LKU8AbQD+1VHeyllQSllOTAdGBMRARwMXF09fwpwVHdrMiBJUhals2lHRIyPiPsajvHruJovATdVt4cACxse66jGVje+JbCsIbitHF8jmxokqRcqpbQBbevz3Ig4C1gB/KSpi+qGAUmSkiidre+yi4gvAp8GRpTyVtvfImC7hmlDqzFWM/4i0D8i+lZZUuP81bJkJ0lZ9OA5pLcTEaOA04AjSylvNDw0Czg2It4dEcOA4cC9wFxgeNVR14+uxodZVSC7HTimev5Y4Lru3t+AJEk1FBHTgF8Du0RER0SMA34AvB+YHREPRMSlAKWUecAM4BHgZmBCKeXNKvs5BbgFmA/MqOYCnA58PSLa6TqndHm3ayob+IdYf3phQetzUNXGpoMPaPUSVDMrli+KZr3WG5d8pWnfl+896d+atq6e4jkkScoiwTmkVrJkJ0lKwQxJkrKo+W7fBiRJysKAJElKwd2+JUlqPTMkScrCkp02pG9++7vc+ct72WJAf3525aUATLr8SmbOupkB/TcH4KtfHsuB++8HwGVTr+KaG26hzyabcOapJ/GxD/8dAFfM+BkzZ91MKYVjjhzF8Z87GoBHH1/AuRf+G2/8/g8M3nYgF0w8jfdttlkLPqk2Fpe1XcThh32KJc+/wF5/OwKAs7/1DY444lA6OwvPL3mBL/3jqTz77HMtXmkN2fatDemoww7h0u+e9xfjx3/uKGZOmcTMKZPeCka/e+IpbppzB9ddeSmXfvc8zv3OD3jzzTd5fMGTzJx1M9N+dDEzp/yQO351L093PAPAxPMv5msnnci1V1zCiAP3599/MrNHP582PlOnzuDwT3/+z8a+c9El7P13h7DPvofyf2/8Od8869QWrU511m1AiogPRsTpEfH96jg9InbticX1BvvstQeb/9X712rubXf9htEjPkG/fv0YOngbPjB0MP9v/m9Z8ORC9th9FzZ9z3vo27cP++y1Bz+/45cAPLVwEfvstQcAH913b2bfcfcG+yzqHe66+x6WvrTsz8ZeffW1t25vttl72dA7uGg1mnj5iY3RGgNSRJxO1wWXgq6N9O6tbk9rvFSt1t20mddz9Akn8c1vf5eXX3kVgCXPv8g2g7Z+a86ggVux5PkX2GmH7fnPB+ex7OVX+P0f/sBdv57L4ueeB2DHYdtz212/BuDW2+9i8XMv9PyHUa9w7jmn88Tv5nLccUfzrbMvbPVy6qmzNO/YCHWXIY0D9i2lnF9KubI6zqfrKoGrvRxt44WhfjR1WjPX2yt87ujDuWnGZGb+eBJbb7kFF/7gsjXO3/GvP8CXPv8Zxp96Fv/96//MLsN3YJNNuv7TnftPpzL9mhv47Je+wutv/J53vcvTglo///y/L2DYjvsybdq1TDj5xFYvRzXU3bdXJzAYeGqV8W2rx95W44Wh3Fz1L221xYC3bh9z5GgmfGMiAAO33vKtzAfguSUvMHDrrQD4hyNG8g9HjATg4kt/zDYDu8Z32H47Lrv42wA8+XQHd/7q3h75DOq9/mPaNVw/6wrOPueiVi+ldkrNu+y6y5C+BsyJiJsioq06bgbmAF/d8MvrnZ5/4b8uYz/njl+x0w7bA/DJj3+Em+bcwfLly+l4ZjFPdzzDHrvuDMCLVc3/2cVLmHPHLznskIP+bLyzs5P/M2U6nz3qsB78JOotdtpp2Fu3jzxiJI899rsWrqbGal6yW2OGVEq5OSJ2pqtEt/J66IuAuaWUNzf04nqDb0w8n7n3P8SyZa8w4qgvcPK445l7/0M89vgCCBiyzSAmnvY/ANhph+0ZefABHPn5L9O3Tx/O+vrJ9OnTB4BT/+k8lr3yCn379uWs/3kyf/X+9wFw4+xfMP2aGwD41Cf25+jDD23NB9VG48orJvGJAz/KVlttwZML7uPsc77D6NEHs/POO9LZ2cnTTy/i5AmeIlbP83pI6lW8HpJ6WjOvh/T6eV9o2vflZt+80ushSZLW00ZaamsWfxgrSUrBDEmSsqh5l50BSZKysGQnSVLrmSFJUhYb6R50zWJAkqQsLNlJktR6ZkiSlETd97IzIElSFpbsJElqPTMkScqi5hmSAUmSsqh527clO0lSCmZIkpSFJTtJUgal5gHJkp0kKQUzJEnKouYZkgFJkrKo+U4NluwkSSmYIUlSFpbsJEkp1DwgWbKTJKVghiRJSZRS7wzJgCRJWViykySp9cyQJCmLmmdIBiRJSsK97CRJSsAMSZKyqHmGZECSpCzqvZWdJTtJUg5mSJKURN2bGgxIkpRFzQOSJTtJUgpmSJKURc2bGgxIkpRE3c8hWbKTJKVghiRJWViykyRlYMlOkqQEzJAkKQtLdpKkDIoBSZKUQs0DkueQJEkpmCFJUhKW7CRJOdQ8IFmykySlYIYkSUlYspMkpVD3gGTJTpJqKiK+GhEPR8S8iPhaNbZFRMyOiMervwdU4xER34+I9oh4KCL2bnidsdX8xyNi7Pqux4AkSUmUzuYd3YmIDwH/DdgP2BP4dETsBJwBzCmlDAfmVPcBRgPDq2M8cEn1OlsAE4EPV681cWUQW1cGJEnKokTzju7tCtxTSnmjlLICuAP4e2AMMKWaMwU4qro9BphauvwG6B8R2wIjgdmllKWllJeA2cCo9fn4BiRJqqeHgQMiYsuIeC9wGLAdMKiU8mw1ZzEwqLo9BFjY8PyOamx14+vMpgZJSqKZTQ0RMZ6u0tpKbaWUtrfeq5T5EXEBcCvwOvAA8OafraeUEhE9dk0MA5IkJVE616rUtnav1RV82rqZczlwOUBEfJuu7Oa5iNi2lPJsVZJbUk1fRFcGtdLQamwRcNAq479YnzVbspOkmoqIgdXfH6Dr/NF/ALOAlZ1yY4HrqtuzgBOqbruPAC9Xpb1bgEMjYkDVzHBoNbbOzJAkKYkW/A5pZkRsCfwJmFBKWRYR5wMzImIc8BTw2WrujXSdZ2oH3gBOBCilLI2Ic4G51bxzSilL12cxBiRJSqKsXXdcE9+vHPA2Yy8CI95mvAATVvM6k4HJ73Q9luwkSSmYIUlSEnXfOsiAJElJNLPLbmNkyU6SlIIZkiQlUXrsJ6g5GZAkKQlLdpIkJWCGJElJ1D1DMiBJUhJ1P4dkyU6SlIIZkiQlYclOkpRCT+9ll40lO0lSCmZIkpSEe9lJklLotGQnSVLrmSFJUhJ1b2owIElSEnVv+7ZkJ0lKwQxJkpKo+9ZBBiRJSsKSnSRJCZghSVISdf8dkgFJkpKoe9u3JTtJUgpmSJKUhF12kqQU6n4OyZKdJCkFMyRJSqLuTQ0GJElKou7nkCzZSZJS2OAZ0qaDD9jQbyFJvULdmxos2UlSEnU/h2TJTpKUghmSJCVhyU6SlELNm+wMSJKURd0zJM8hSZJSMEOSpCTq3mVnQJKkJDpbvYAWs2QnSUrBDEmSkihYspMkJdBZ875vS3aSpBTMkCQpiU5LdpKkDOp+DsmSnSQpBTMkSUqi7r9DMiBJUhKW7CRJSsAMSZKSsGQnSUqh7gHJkp0kKQUzJElKou5NDQYkSUqis97xyJKdJCkHMyRJSsK97CRJKdT86hOW7CRJOZghSVISdf8dkgFJkpLojHqfQ7JkJ0lKwQxJkpKoe1ODAUmSkqj7OSRLdpKkFMyQJCmJum8dZECSpCTqvlODJTtJqqmI6B8RV0fEoxExPyI+GhFbRMTsiHi8+ntANTci4vsR0R4RD0XE3g2vM7aa/3hEjF3f9RiQJCmJ0sRjLf0rcHMp5YPAnsB84AxgTillODCnug8wGhheHeOBSwAiYgtgIvBhYD9g4sogtq4MSJKURGc07+hORGwOHAhcDlBKWV5KWQaMAaZU06YAR1W3xwBTS5ffAP0jYltgJDC7lLK0lPISMBsYtT6f34AkSfU0DHge+PeIuD8ifhQRmwGDSinPVnMWA4Oq20OAhQ3P76jGVje+zgxIkpREZxOPiBgfEfc1HONXebu+wN7AJaWUvwVe57/KcwCUUtaxAvjO2GUnSUk085u/lNIGtK1hSgfQUUq5p7p/NV0B6bmI2LaU8mxVkltSPb4I2K7h+UOrsUXAQauM/2J91myGJEk1VEpZDCyMiF2qoRHAI8AsYGWn3Fjguur2LOCEqtvuI8DLVWnvFuDQiBhQNTMcWo2tMzMkSUqiBT+M/Qrwk4joBywATqQrUZkREeOAp4DPVnNvBA4D2oE3qrmUUpZGxLnA3GreOaWUpeuzGAOSJCXR03vZlVIeAPZ5m4dGvM3cAkxYzetMBia/0/VYspMkpWCGJElJ1H23bwOSJCVR6r2VnSU7SVIOZkiSlIQlO0lSCnUPSJbsJEkpmCFJUhI9tmlcUgYkSUqi7pcwt2QnSUrBDEmSkqh7U4MBSZKSqHtAsmQnSUrBDEmSkrDLTpKUQt277AxIkpSE55AkSUrADEmSkvAckiQphc6ahyRLdpKkFMyQJCmJujc1GJAkKYl6F+ws2UmSkjBDkqQkLNlJklKo+04NluwkSSmYIUlSEnX/HZIBSZKSqHc4smQnSUrCDEmSkrDLTpKUQt3PIVmykySlYIYkSUnUOz8yIElSGnU/h2TJTpKUghmSJCVR96YGA5IkJVHvcGTJTpKUhBmSJCVR96YGA5IkJVFqXrSzZCdJSsEMSZKSsGQnSUqh7m3fluwkSSmYIUlSEvXOjwxIkpSGJTulclnbRTzT8SAP3D+n1UtRLzV06GB+futPeejB23nwgdv4yinjABgwoD833ziN+fPu5uYbp9G//+YtXqnqxoCUzNSpMzj8059v9TLUi61YsYJvnHY2f7PnJ/nYx4/gpJO+yK67Duf00yZw2+13s+vuH+e22+/m9NMmtHqptdPZxGNjZEBK5q6772HpS8tavQz1YosXL+H+Bx4G4LXXXufRRx9nyOBtOOKIkUy94qcATL3ipxx55KhWLrOWShP/bIzWOyBFxInNXIiknrf99kPZa88Pcc+99zNo4FYsXrwE6ApagwZu1eLVqW7eSYZ0dtNWIanHbbbZe5lx1WV8/X9N5NVXX/uLx0vZOP8ve2NW95LdGrvsIuKh1T0EDFrD88YD4wGiz+Zssslm671ASc3Xt29ffnrVZUybdi0/+9lNADy35AW22WYgixcvYZttBrLk+RdbvMr62VhLbc3SXYY0CDgBOOJtjtX+ay2ltJVS9iml7GMwkvK5rO0i5j/azsX/2vbW2A3X38oJx38GgBOO/wzXX39Lq5anmuouIN0AvK+U8tQqx5PALzb46mroyismcfeds9hl5x15csF9nPjFY1u9JPUyH9t/X47/wjF88pP7c9/cW7lv7q2MHnUwF1w4iU+NOJD58+5mxMEHcMG/TGr1Umun7iW72NB14r79htQ7B5XUq61Yviia9VrHb//3Tfu+vOKpa5q2rp5i27ckKQW3DpKkJOpeTjIgSVIS7mUnSVICZkiSlETdf4dkQJKkJDbWdu1msWQnSUrBDEmSkqh7U4MBSZKSqPs5JEt2kqQUzJAkKYm6NzUYkCQpibpfg8qSnSTVUES8JyLujYgHI2JeRJxdjQ+LiHsioj0iroqIftX4u6v77dXjf93wWmdW449FxMj1XZMBSZKS6KQ07VgLfwQOLqXsCewFjIqIjwAXAN8rpewEvASMq+aPA16qxr9XzSMidgOOBXYHRgE/jIg+6/P5DUiSlERPXg+pdFl57fp3VUcBDgaursanAEdVt8dU96keHxERUY1PL6X8sZTyBNAO7Leunx0MSJKURmnin4gYHxH3NRzjV32/iOgTEQ8AS4DZwO+AZaWUFdWUDmBIdXsIsBCgevxlYMvG8bd5zjqxqUGSeqFSShvQ1s2cN4G9IqI/cC3wwZ5Y2+oYkCQpiVbt1FBKWRYRtwMfBfpHRN8qCxoKLKqmLQK2Azoioi+wOfBiw/hKjc9ZJ5bsJCmJUkrTju5ExNZVZkREbAocAswHbgeOqaaNBa6rbs+q7lM9flvpeqNZwLFVF94wYDhw7/p8fjMkSaqnbYEpVUfcJsCMUsoNEfEIMD0izgPuBy6v5l8OXBER7cBSujrrKKXMi4gZwCPACmBCVQpcZ7Ghf4jVt9+Qev/SS1KvtmL5omjWa43cbnTTvi9vWXhT09bVU8yQJCkJN1eVJCkBMyRJSsLrIUmSUnBzVUmSEjBDkqQkLNlJklKwy06SpATMkCQpic6aNzUYkCQpiXqHI0t2kqQkzJAkKQm77CRJKdQ9IFmykySlYIYkSUnUfesgA5IkJWHJTpKkBMyQJCmJum8dZECSpCTqfg7Jkp0kKQUzJElKou5NDQYkSUrCkp0kSQmYIUlSEpbsJEkp1L3t25KdJCkFMyRJSsIrxkqSUrBkJ0lSAmZIkpSEJTtJUgqW7CRJSsAMSZKSsGQnSUrBkp0kSQmYIUlSEpbsJEkpWLKTJCkBMyRJSqKUzlYvoaUMSJKURN2vh2TJTpKUghmSJCVR7LKTJGVgyU6SpATMkCQpCUt2kqQU6r5TgyU7SVIKZkiSlETdtw4yIElSEp5DkiSlYNu3JEkJmCFJUhKW7CRJKdj2LUlSAmZIkpSEJTtJUgp22UmSlIAZkiQlYclOkpSCXXaSJCVghiRJSbi5qiQpBUt2kiQlYIYkSUnYZSdJSqHu55As2UmSUjAgSVISpZSmHWsjIkZFxGMR0R4RZ2zgj9ctS3aSlERPnkOKiD7AJOAQoAOYGxGzSimP9NgiVmGGJEn1tB/QXkpZUEpZDkwHxrRyQQYkSUqiNPFYC0OAhQ33O6qxltngJbsVyxfFhn6P3igixpdS2lq9DtWH/+Zar5nflxExHhjfMNSW/b+vGVJe47ufIjWV/+Z6kVJKWylln4Zj1WC0CNiu4f7QaqxlDEiSVE9zgeERMSwi+gHHArNauSC77CSphkopKyLiFOAWoA8wuZQyr5VrMiDllbrWq17Jf3M1U0q5Ebix1etYKeq+d5IkKQfPIUmSUjAgJZRtOw/1XhExOSKWRMTDrV6LZEBKpmE7j9HAbsBxEbFba1elXuzHwKhWL0ICA1JG6bbzUO9VSrkTWNrqdUhgQMoo3XYektQTDEiSpBQMSPmk285DknqCASmfdNt5SFJPMCAlU0pZAazczmM+MKPV23mo94qIacCvgV0ioiMixrV6Taovd2qQJKVghiRJSsGAJElKwYAkSUrBgCRJSsGAJElKwYAkSUrBgCRJSsGAJElK4f8DKftbwrShFccAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml_rfeX = RandomForestClassifier(n_jobs=-1, n_estimators=49,  max_features=0.592857, bootstrap=True, max_depth=8)\n",
    "%time ml_rfeX.fit(X_train, y_train)\n",
    "pred_d = ml_rfeX.predict(X_test)\n",
    "print_score(pred_d,ml_rfeX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First Set of results from the random search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 16s, sys: 68 ms, total: 1min 16s\n",
      "Wall time: 5.3 s\n",
      "['roc_auc-->', 0.9983787718369508, 'F1_score-->', 0.4615384615384615, 'roc_auc_score-->', 0.9983787718369508]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.3940298507462686]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.9981415249407353]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.01015\n",
      "Accuracy  :  0.98985\n",
      "Precision  :  0.24535315985130113\n",
      "Recall  :  1.0\n",
      "Specificity  :  1.0\n",
      "False positive rate  :  0.010183605899468245\n",
      "True positive rate  :  1.0\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGltJREFUeJzt3XucXWV18PHfmgmhCC0JIQRyEUKJFbRKEQGlWCAVkoAGkCK8XKIvGisXUSsCio0U7AtvK7e3ERxKJIACKahEyj1QoZaEoFwkCZARhEwgBAgoFzUm8/SP2UnnjSSThJM5K7N/Xz77k3Oe85wzz4F8ZrHWXvvZUUpBkqRma2n2AiRJAgOSJCkJA5IkKQUDkiQpBQOSJCkFA5IkKQUDkiQpBQOSJNVQREyJiMUR8Wi3sa9HxMKIeKg6xnV77YyIaI+IxyPiwG7jY6qx9og4vdv4yIiYVY1fFxH9e1qTAUmS6ukKYMybjF9QStm1Om4GiIhdgCOBd1Xv+VZEtEZEKzAZGAvsAhxVzQU4r/qsnYCXgeN7WlC/t/iFevT7F590Kwj1ms2G7tPsJahmli1dGI36rEb+vtxk6x3XuK5Syj0RscNaftx44NpSyu+ApyKiHdijeq29lPIkQERcC4yPiHnA/sD/quZMBb4OXLKmH2KGJEnq7qSIeKQq6Q2sxoYBC7rN6ajGVjc+CHillLJslfE1MiBJUhadyxt2RMTEiHig2zFxLVZwCfCnwK7Ac8A3N+j3XcUGL9lJknpfKaUNaFvH9zy/4nFEXAbcVD1dCIzoNnV4NcZqxl8CBkREvypL6j5/tcyQJCmL0tm4Yz1ExHbdnh4KrOjAmw4cGRGbRsRIYBRwPzAbGFV11PWnq/Fheum6jcTdwOHV+ycAN/b0882QJCmLzvULJOsjIq4B9gW2jogOYBKwb0TsChTgl8BnAEopcyJiGjAXWAacWEpZXn3OScBtQCswpZQyp/oRpwHXRsQ5wIPA5T2uaUPfD8kuO/Umu+zU2xraZffcvMZ12W23c8PW1VvMkCQpibKepba+woAkSVn0YskuI5saJEkpmCFJUhaW7CRJKXQub/YKmsqSnSQpBTMkScrCkp0kKQW77CRJaj4zJElKwgtjJUk5WLKTJKn5zJAkKQtLdpKkFLwwVpKk5jNDkqQsLNlJklKwy06SpOYzQ5KkLCzZSZJSsGQnSVLzmSFJUhKl1Ps6JAOSJGVR83NIluwkSSmYIUlSFjVvajAgSVIWNS/ZGZAkKQs3V5UkqfnMkCQpC0t2kqQUat7UYMlOkpSCGZIkZWHJTpKUgiU7SZKazwxJkrKoeYZkQJKkJOq+27clO0lSCmZIkpSFJTtJUgo1b/u2ZCdJSsEMSZKysGQnSUrBkp0kSc1nhiRJWViykySlYMlOkqTmM0OSpCws2UmSUqh5QLJkJ0lKwQxJkrKoeVODAUmSsrBkJ0lS85khSVIWluwkSSlYspMkqfnMkCQpC0t2kqQULNlJktR8ZkiSlEXNMyQDkiRlUUqzV9BUluwkSSmYIUlSFpbsJEkp1DwgWbKTJKVghiRJWXhhrCQpBUt2kiQ1nwFJkrIopXFHDyJiSkQsjohHu439U0Q8FhGPRMQPImJAt9fOiIj2iHg8Ig7sNj6mGmuPiNO7jY+MiFnV+HUR0b+nNRmQJCmLzs7GHT27AhizytgdwLtLKe8BngDOAIiIXYAjgXdV7/lWRLRGRCswGRgL7AIcVc0FOA+4oJSyE/AycHxPCzIgSVINlVLuAZasMnZ7KWVZ9XQmMLx6PB64tpTyu1LKU0A7sEd1tJdSniylLAWuBcZHRAD7A9dX758KHNLTmmxqkKQscjU1/G/guurxMLoC1Aod1RjAglXG9wQGAa90C27d56+WAUmSsmhg23dETAQmdhtqK6W0reV7vwosA77bsAWtBQOSJPVBVfBZqwDUXUR8AjgYGF3Kyu6IhcCIbtOGV2OsZvwlYEBE9KuypO7zV8tzSJKUROksDTvWR0SMAb4MfLSU8ka3l6YDR0bEphExEhgF3A/MBkZVHXX96Wp8mF4FsruBw6v3TwBu7OnnmyFJUha9eA4pIq4B9gW2jogOYBJdXXWbAnd09SUws5Tyt6WUORExDZhLVynvxFLK8upzTgJuA1qBKaWUOdWPOA24NiLOAR4ELu9xTWUD33/j9y8+We8bfKhXbTZ0n2YvQTWzbOnCaNRnvXHpKQ37ffm2v72oYevqLWZIkpSFe9lJklJYz3M/fYVNDZKkFMyQJCmLXBfG9joDkiRlYUCSJKWwgbues/MckiQpBTMkScrCkp02pDP/8Xzu+cn9bDVwAD+8+lIAJl9+NTdMv5WBA7YE4JTPTOBDH9wDgMuuvI7v33QbrS0tnPGFz7L3nu8D4Nevvsakcy+k/cmnIYKzv/IFdn33zvy/tiu56z/voyVa2Grglnzjq3/HNoMHNefLaqPS0tLCrJm38OzCRYw/dAL77bs35533Nfr334Sf/eznfHri37F8+fJmL7NebPvWhnTIuA9z6fnn/MH4sR8/hBumTuaGqZNXBqNfPPU0t8z4MTdefSmXnn8OZ//zv6z8hXDuhZey956786NrLuP7Uyez4/Zd+xl+8uiP8YMrL+GGqZP5q7335JLvfK/3vpw2ap87+VM89th8ACKCKZdfyNHHnMCufzGaZ57p4Lhj/6bJK1Td9BiQIuKdEXFaRFxcHadFxM69sbi+YPdd/5wt/+SP12ruXffOZOzov6J///4MH7otbx8+lJ/Pe4JXX3udnz78KB/7SNddgzfZZBP+5I+3AGCLzTdf+f7f/Oa3xEa3WYiaYdiw7Rg3djRTplwDwKBBA1m6dCnz5z8JwJ133sNhh45r5hLrqXQ27tgIrTEgRcRpdN0BMOja2fX+6vE13e+drnV3zQ0/4tDjPsuZ/3g+v/r1qwAsfuElth0yeOWcIdtszeIXXmThs4sYOGBLzvzG+Rz+iRP5+/9zIW/85rcr51307SsYfeix/Pvtd3PSp47t9e+ijc/53zyL0884h87qnMWLLy6hX79+vG+39wBw2GEHMXzE0GYusZ46S+OOjVBPGdLxwPtLKeeWUq6ujnPpum3tau+PHhETI+KBiHjgX6+8ppHr7RM+fuhB3DJtCjdcMZnBg7bin/7lsjXOX7Z8OfOeaOfjhx7E9VdMZrPN/ojLr5q28vVTPvMJZvzgKg46YD++d8OPNvTytZE7aNxfs3jxi/zswZ//f+NHH3MC3/znr3PfT27itddeZ/nyjfP/srXx6ikgdQJv9r9J21WvvalSSlspZfdSyu6fOu6ot7K+PmnrrQbS2tpKS0sLh390LI/OfQKAbQYPYtHzL6yc9/ziF9lm8NZsu83WDBm8Ne951zsBOGDfv2TuE+1/8LkHH7Afd/7HT3rnS2ij9cEP7s5HDj6A9idm8t2rv8V+++3N1CsuZuasn7Lv/ofxgb0P5t57Z64s36n3lM7Ohh0bo54C0ueBGRFxS0S0VcetwAzglA2/vL7phReXrHw848f/xU47bg/Afn+5F7fM+DFLly6l49lFPNPxLH++8zvYetBWbLvNYJ56ugOAmT99iD/d4e0APL3gf27CeNe99zFy++G9+E20Mfrqmeeyw467s9M79uLoY07g7rt/woRPfI7BVXdm//79OfVLJ9LWdlWTV1pDNS/ZrbHtu5Rya0S8g64S3bBqeCEwe8XNmbRmp046l9kPPsIrr/ya0YccwwnHH8vsBx/h8flPQsCwbYcw6cufA2CnHbfnwP334aNHf4Z+ra189Ysn0NraCsBXvvBZTjvr//L7Zb9nxNDtOPsrXwDggku+wy+f6SBagqHbbsPfn3py076rNm5f+uJnGXfQX9PS0sK3v30ld5ttq5d5gz71Kd6gT72tkTfoe/2cYxr2+3LzM6/e6HpuvTBWkrLYSEttjeKFsZKkFMyQJCmLjbQ7rlEMSJKUhSU7SZKazwxJkrLYSPegaxQDkiRlYclOkqTmM0OSpCQ21j3oGsWAJElZWLKTJKn5zJAkKYuaZ0gGJEnKouZt35bsJEkpmCFJUhaW7CRJGZSaByRLdpKkFMyQJCmLmmdIBiRJyqLmOzVYspMkpWCGJElZWLKTJKVQ84BkyU6SlIIZkiQlUUq9MyQDkiRlYclOkqTmM0OSpCxqniEZkCQpCfeykyQpATMkScqi5hmSAUmSsqj3VnaW7CRJOZghSVISdW9qMCBJUhY1D0iW7CRJKZghSVIWNW9qMCBJUhJ1P4dkyU6SlIIZkiRlYclOkpSBJTtJkhIwQ5KkLCzZSZIyKAYkSVIKNQ9InkOSJKVghiRJSViykyTlUPOAZMlOkpSCGZIkJWHJTpKUQt0DkiU7SaqpiDglIh6NiDkR8flqbKuIuCMi5ld/DqzGIyIujoj2iHgkInbr9jkTqvnzI2LC+q7HgCRJSZTOxh09iYh3A58G9gDeCxwcETsBpwMzSimjgBnVc4CxwKjqmAhcUn3OVsAkYM/qsyatCGLryoAkSVmUaNzRs52BWaWUN0opy4AfA4cB44Gp1ZypwCHV4/HAlaXLTGBARGwHHAjcUUpZUkp5GbgDGLM+X9+AJEl9UERMjIgHuh0TV5nyKLBPRAyKiLcB44ARwJBSynPVnEXAkOrxMGBBt/d3VGOrG19nNjVIUhKNbGoopbQBbWt4fV5EnAfcDrwOPAQsX2VOiYheuyeGGZIkJVE6o2HHWv28Ui4vpbyvlPIh4GXgCeD5qhRH9efiavpCujKoFYZXY6sbX2cGJEmqqYjYpvrz7XSdP/oeMB1Y0Sk3AbixejwdOK7qttsL+FVV2rsNOCAiBlbNDAdUY+vMkp0kJdGE65BuiIhBwO+BE0spr0TEucC0iDgeeBo4opp7M13nmdqBN4BPApRSlkTE2cDsat4/lFKWrM9iDEiSlERZu+64Bv68ss+bjL0EjH6T8QKcuJrPmQJMeavrsWQnSUrBDEmSkqj71kEGJElKYm274/oqS3aSpBTMkCQpidJrl6DmZECSpCQs2UmSlIAZkiQlUfcMyYAkSUnU/RySJTtJUgpmSJKUhCU7SVIKvb2XXTaW7CRJKZghSVIS7mUnSUqh05KdJEnNZ4YkSUnUvanBgCRJSdS97duSnSQpBTMkSUqi7lsHGZAkKQlLdpIkJWCGJElJ1P06JAOSJCVR97ZvS3aSpBTMkCQpCbvsJEkp1P0ckiU7SVIKZkiSlETdmxoMSJKURN3PIVmykySlsMEzpM2G7rOhf4Qk9Ql1b2qwZCdJSdT9HJIlO0lSCmZIkpSEJTtJUgo1b7IzIElSFnXPkDyHJElKwQxJkpKoe5edAUmSkuhs9gKazJKdJCkFMyRJSqJgyU6SlEBnzfu+LdlJklIwQ5KkJDot2UmSMqj7OSRLdpKkFMyQJCmJul+HZECSpCQs2UmSlIAZkiQlYclOkpRC3QOSJTtJUgpmSJKURN2bGgxIkpREZ73jkSU7SVIOZkiSlIR72UmSUqj53Scs2UmScjBDkqQk6n4dkgFJkpLojHqfQ7JkJ0lKwQxJkpKoe1ODAUmSkqj7OSRLdpKkFMyQJCmJum8dZECSpCTqvlODJTtJqqmIGBAR10fEYxExLyI+EBFbRcQdETG/+nNgNTci4uKIaI+IRyJit26fM6GaPz8iJqzvegxIkpREaeCxli4Cbi2lvBN4LzAPOB2YUUoZBcyongOMBUZVx0TgEoCI2AqYBOwJ7AFMWhHE1pUBSZKS6IzGHT2JiC2BDwGXA5RSlpZSXgHGA1OraVOBQ6rH44ErS5eZwICI2A44ELijlLKklPIycAcwZn2+vwFJkvqgiJgYEQ90OyauMmUk8ALwnYh4MCL+NSI2B4aUUp6r5iwChlSPhwELur2/oxpb3fg6s6lBkpJo5HVIpZQ2oG0NU/oBuwEnl1JmRcRF/E95bsVnlIjotet1zZAkKYlePofUAXSUUmZVz6+nK0A9X5XiqP5cXL2+EBjR7f3Dq7HVja8zA5Ik1VApZRGwICL+rBoaDcwFpgMrOuUmADdWj6cDx1XddnsBv6pKe7cBB0TEwKqZ4YBqbJ1ZspOkJJpwYezJwHcjoj/wJPBJuhKVaRFxPPA0cEQ192ZgHNAOvFHNpZSyJCLOBmZX8/6hlLJkfRZjQJKkJHp7L7tSykPA7m/y0ug3mVuAE1fzOVOAKW91PZbsJEkpmCFJUhJ13+3bgCRJSZR6b2VnyU6SlIMZkiQlYclOkpRC3QOSJTtJUgpmSJKURK9tGpeUAUmSkqj7Lcwt2UmSUjBDkqQk6t7UYECSpCTqHpAs2UmSUjBDkqQk7LKTJKVQ9y47A5IkJeE5JEmSEjBDkqQkPIckSUqhs+YhyZKdJCkFMyRJSqLuTQ0GJElKot4FO0t2kqQkzJAkKQlLdpKkFOq+U4MlO0lSCmZIkpRE3a9DMiBJUhL1DkeW7CRJSZghSVISdtlJklKo+zkkS3aSpBTMkCQpiXrnRwYkSUqj7ueQLNlJklIwQ5KkJOre1GBAkqQk6h2OLNlJkpIwQ5KkJOre1GBAkqQkSs2LdpbsJEkpmCFJUhKW7CRJKdS97duSnSQpBTMkSUqi3vmRAUmS0rBkp3QOPGBf5jx6D4/N/U++fOqJzV6O+pjhw4dy5+3/xiMP383DD93FyScdD8DHPnYwDz90F0t/u4D37faeJq9SdWSGlExLSwsXX/QNxow7io6O55h538386KbbmTdvfrOXpj5i2bJlnPrls3jwoUfZYovNuX/Wrdw54x7mzHmMvzni01wy+dxmL7G27LJTKnu8/y/4xS9+yVNPPQPAtGk38tGPHGhAUsMsWrSYRYsWA/Daa6/z2GPzGTZ0W+6ccW+TVyYvjF1PEfHJRi5EXYYO25YFHc+ufN6x8DmGDt22iStSX7b99sPZ9b3vZtb9DzZ7KdJbOod0VsNWIanXbb7525h23WV88UuTePXV15q9HNFVsmvUsTFaY8kuIh5Z3UvAkDW8byIwESBat6SlZfP1XmDdPLtwESOGD135fPiw7Xj22UVNXJH6on79+vFv113GNdf8gB/+8JZmL0eVupfsejqHNAQ4EHh5lfEA/mt1byqltAFtAP36D6v3v+F1NPuBh9hpp5HssMMIFi5cxBFHjOfY4+y0U2Nd1vZN5j3WzoUXtTV7KdJKPQWkm4AtSikPrfpCRPzHBllRzS1fvpxTPn8mN//792htaeGKqdcxd+4TzV6W+pC9P/h+jj3mcB75+VwemH07AF/72rn037Q/F11wDoMHb8X0G6/k4YfnMO7go5u82nrZWEttjRKlbNgExgxJUl+2bOnCaNRnHbv9YQ37fXnV099v2Lp6ixfGSpJS8DokSUqi7uUkA5IkJeFedpIkJWCGJElJeB2SJCmFurd9W7KTJKVghiRJSdS9qcGAJElJ1P0ckiU7SVIKZkiSlIRNDZKkFEopDTt6EhF/FBH3R8TDETEnIs6qxkdGxKyIaI+I6yKifzW+afW8vXp9h26fdUY1/nhEHLi+39+AJEn19Dtg/1LKe4FdgTERsRdwHnBBKWUnum49dHw1/3jg5Wr8gmoeEbELcCTwLmAM8K2IaF2fBRmQJCmJTkrDjp6ULituFbxJdRRgf+D6anwqcEj1eHz1nOr10RER1fi1pZTflVKeAtqBPdbn+xuQJCmJ3r6FeUS0RsRDwGLgDuAXwCullGXVlA5gWPV4GLAAoHr9V8Cg7uNv8p51YkCSpCRKA/+JiIkR8UC3Y+If/LxSlpdSdgWG05XVvLPXv3Q3dtlJUh9USmkD1uoe9aWUVyLibuADwICI6FdlQcOBhdW0hcAIoCMi+gFbAi91G1+h+3vWiRmSJCXRm+eQImJwRAyoHm8GfBiYB9wNHF5NmwDcWD2eXj2nev2u0tXONx04surCGwmMAu5fn+9vhiRJSaxNu3YDbQdMrTriWoBppZSbImIucG1EnAM8CFxezb8cuCoi2oEldHXWUUqZExHTgLnAMuDEUsry9VlQbOh/Af36D6v3XhiS+rRlSxdGoz5r7IixDft9ecuCWxq2rt5ihiRJSdR9pwYDkiQl4eaqkiQlYIYkSUl4PyRJUgq93GWXjiU7SVIKZkiSlIQlO0lSCnbZSZKUgBmSJCXRWfOmBgOSJCVR73BkyU6SlIQZkiQlYZedJCmFugckS3aSpBTMkCQpibpvHWRAkqQkLNlJkpSAGZIkJVH3rYMMSJKURN3PIVmykySlYIYkSUnUvanBgCRJSViykyQpATMkSUrCkp0kKYW6t31bspMkpWCGJElJeMdYSVIKluwkSUrADEmSkrBkJ0lKwZKdJEkJmCFJUhKW7CRJKViykyQpATMkSUrCkp0kKQVLdpIkJWCGJElJlNLZ7CU0lQFJkpKo+/2QLNlJklIwQ5KkJIpddpKkDCzZSZKUgBmSJCVhyU6SlELdd2qwZCdJSsEMSZKSqPvWQQYkSUrCc0iSpBRs+5YkKQEzJElKwpKdJCkF274lSUrADEmSkrBkJ0lKwS47SZISMEOSpCQs2UmSUrDLTpKkBMyQJCkJN1eVJKVgyU6SpATMkCQpCbvsJEkp1P0ckiU7SVIKBiRJSqKU0rBjbUTEmIh4PCLaI+L0Dfz1emTJTpKS6M1zSBHRCkwGPgx0ALMjYnopZW6vLWIVZkiSVE97AO2llCdLKUuBa4HxzVyQAUmSkigNPNbCMGBBt+cd1VjTbPCS3bKlC2ND/4y+KCImllLamr0O1Yd/55qvkb8vI2IiMLHbUFv2/75mSHlN7HmK1FD+netDSiltpZTdux2rBqOFwIhuz4dXY01jQJKkepoNjIqIkRHRHzgSmN7MBdllJ0k1VEpZFhEnAbcBrcCUUsqcZq7JgJRX6lqv+iT/ztVMKeVm4OZmr2OFqPveSZKkHDyHJElKwYCUULbtPNR3RcSUiFgcEY82ey2SASmZbtt5jAV2AY6KiF2auyr1YVcAY5q9CAkMSBml285DfVcp5R5gSbPXIYEBKaN023lIUm8wIEmSUjAg5ZNuOw9J6g0GpHzSbechSb3BgJRMKWUZsGI7j3nAtGZv56G+KyKuAe4D/iwiOiLi+GavSfXlTg2SpBTMkCRJKRiQJEkpGJAkSSkYkCRJKRiQJEkpGJAkSSkYkCRJKRiQJEkp/DfNbHFmzGWHJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml_a = RandomForestClassifier(n_jobs=-1, n_estimators=65,  max_features=0.66, bootstrap=True, max_depth=5)\n",
    "%time ml_a.fit(X_train, y_train)\n",
    "pred_d = ml_a.predict(X_test)\n",
    "print_score(pred_d,ml_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Importances\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGfCAYAAABFpjj0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xu8nVV97/vPtwEiKITDxe5AOaZKqoeLZEsQUQqKVMtON5eKRV52S9i6OexqtcfamqOevvBY9wm621KLVvEChU2FoqWosSBWbiKIK5CQoKK1BhRpCyIRSDQEf+ePOZZMV1bWNVkzD+vzfr3yWs8cczxj/J5nLczX8Yy5kqpCkiSpi35p0AVIkiRNlUFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR11k6DLkDaZ599asGCBYMuQ5K0A1m5cuWDVbXveP0MMhq4BQsWMDQ0NOgyJEk7kCT3TKSfj5YkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJn+QvxNHBr7lvPgmUrRn1v3fIlM1yNJKlLXJGRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdNeuCTJInkqxKsjbJFUl2m8ZYS5OcP41z9+t7vXOS5Um+neT2JLckOWGcMf5gOvVvD0lOSnJnu8dDSY4edE2SpKeuWRdkgI1VtaiqDgE2AWf3v5membgvS4H9+l6/B5gPHFJVLwBOBnYfZ4w/ALZrkEky2d819E/AYVW1CPivwMe2fVWSJPXMxiDT7ybgwCQLktyd5GJgLXBAktOTrGkrN+cOn5DkzCTfSnIb8JK+9ouSnNr3+tG+47e3sVa3VZdTgcXApW3l4unAfwN+v6p+ClBV/1ZVf9fO/+u2unFXkne3tjfTC0LXJbmutb2ireTc3labntHa/1OSbyZZmeQDST7X2vdK8g9tBeXWJM9v7eckuSTJzcAlSW5Msqjver6c5LDRbmhVPVpV1V4+HajR+iU5q13T0BMb1o/7jZIkaTSzNsi0lYYTgDWtaSHwoao6GHgcOBc4DlgEHJHk5CTzgXfTCzBHAwdNYJ4TgJOAI6vqMOB9VfUpYAh4bVu5eA5wb1X9eCvDvLOqFgPPB45N8vyq+gDwA+BlVfWyJPsA7wKObys6Q8BbkzwN+AhwQlUdDuzbN+67gTuq6vnAO4CL+947qI11OvBxeitIJPk14GlVtXqMaz4lyTeBFfRWZbZQVRdU1eKqWjxnt3lbG0qSpDHNxiCza5JV9P6iv5feX9IA91TVre34COD6qnqgqjYDlwLHAEf2tW8CLp/AfMcDF1bVBoCqemgKNf9OktuBO4CDGT1Avai139yu7wzgWcDzgH+pqu+2fp/sO+do4JJW15eAvZPs0d77TFVtbMdXAL+VZGd6weSisYqtqiur6nn0Ho+9ZzIXKknSZMzGf2tpY1sF+bkkAI9Nc9zNtGDY9tjsMolz/xn435PsMXJVJsmvAm8DjqiqHyW5CHjaKGMEuLatoPSfv2iUvhPx8/tRVRuSXEtvZel3gMMnMkBV3Zjk2Un2qaoHp1iHJElbNRtXZCbiNnqPcPZJMgc4HbgB+Gpr37utTry675x1PPkX/InAzu34WuDM4U8XJdmrtT9C28zbVms+Dvxlkl1av32TvBrYg16oWJ/kl+k9DmPkGMCtwEuSHNjOf3p7DHQ38OwkC1q/0/rOvwl4bev/UuDBMR5vfQz4APC1qvrRVvqQ5MC0ZJjkBcBc4Idb6y9J0nTMxhWZcVXV/UmWAdfRW+lYUVVXQW8jLHAL8DCwqu+0jwJXJVkNXE1b0aiqq9uqyFCSTcDn6e1HuQj4cJKNwFH09rf8KfD1JD9p5/9JVa1OcgfwTeB7wM19c14AXJ3kB22fzFLgk0nmtvffVVXfSvJ7rd9jwNf6zj8H+ESSO4EN9B5Hbe2erEzyY+DCcW7fq4DXJXkc2Aic1rf5V5KkbSr+HfPUl+QZVfVoWyn5IPDtqvqLSY6xH3A98Lyq+tm2rG/u/IU1/4zzRn1v3fIl23IqSVJHJFnZPugyJh8tzQ7/rW0AvguYR+9TTBOW5HX0Hqu9c1uHGEmSpsNHS7NAW32Z1ArMiPMv5hc/mk2SM4G3jOh6c1W9carzSJI0WQYZTUlVXcj4+2UkSdquDDIauEP3n8eQe2EkSVPgHhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZOw26AGnNfetZsGzFVt9ft3zJDFYjSeoSV2QkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnjRlkkuydZFX7869J7ut7vctMFTlGfb+d5Hl9r9+b5GVTHGt+ks8nWZ3k60k+09qfneQ1UxhvryRnt+NFffftoSTfbcfXJDkgyeVTqXmUOf90xPdoVZLdkxyf5B+2xRyTrOfAJKtmel5J0uwx5sevq+qHwCKAJOcAj1bV/+zvkyRAqupn26vIMfw28DPgmwBV9c5pjPWnwIqq+iBAkue39mcDrwEum+hASXYC9gLOBj5cVat48j7+L+BTVdUfLE6bRt0jvb+qzhtRzzYcXpKkHceUHi21/6f99SSXAncB85NckGQoyV1J/qSv7/eTnJPkjiR3Jvm11n5cW/1YleT2JE9PskeSL7XXdyb5rb5xzmxtq5NcmOTXgf8E/EUbY0GS/5Xk5Nb/Fa19TZKPDq8gba0eYD7w/eH5qurOdrgceFkb681JnpPkpnb+yiRHtnGPT3J9ks8Ba9p5z23nLR/nXq5qx29I8vdJvpjkniT/Pckftbm+kmTP1m9hW81ZmeTGvmuYyPfuRUluaWPenGRh39xXJrkhybeTvKu1757kH9t9X5vk1NZ+ROu7sr3/y33td7ZrOnuidUmSNBXT+YV4zwNeV1VDAEmWVdVDbTXiuiSfqqqvt77/VlX/McmbgbfS+wvuj4CzquqrSZ4B/IResDq5qn6c5JnAzcDnkhwGvB14cZtjr/b18/StbgyvPCTZDfgEcGxVfacFrrOA88eo53zgb5PcDnwRuLCq7geWAW+qqpP7xv6NqvpJeo+1/gY4so27GDioqu5NciBwYFUtmuR9PRh4AfAM4NvAW1utfwX8bqvzAuAN7dpe0tpe0c7/oyRL2/GDVXX8iPG/Afx6VW1O8pv0VqKGV4ReCBwCbAK+1kLZc4F1VXVCu/55SeYCfwmcWFUPJnkt8J52jy+i9329OclfbO0ik5zV+jNnj30neYskSeqZTpD5znCIaU5P8vo25n7AQcBwkPn79nUlvVUU6IWUv2wh49NV9WiSOcDyJEfTe2R0QJJ9gOOAy6vqIYDhr2P4P4BvVdV32uuLgdfzZJDZop6q+nyS5wC/CZwA3JHk4FHGnguc38LVZuA5fe/dUlX3jlPbeL5UVY8BjyV5FPhsa18D/FpblXkR8Ok8+cio//u4xaOlEfYELm7XOtI1VfUjgPT21BwN/BO978ly4LMtoCyiF7i+2GqYA3y/fa92raqb23iXAKPuWaqqC+gFMubOX1hj1CtJ0lZNJ8g8NnzQHk+8BXhhVT2c3j6Qp/X1/Wn7+sTwnFX1p+ltqF0C3Jrk5cCxwDzgBW3F4PsjxtlWtqin1fRD4FLg0iRX0/uL/LER5/4h8D16qyM7A4/2vTey73Rqg16Y+2nf8U5A6K20THalZ9h76QWWD7VVo6v73hsZKKqqvpFkMb3AtzzJPwL/CNxZVb/e37kFGUmSZsy2+vj1HsAjwI+TzAdeOd4JSZ5TVXdW1f8H3E7vEcY84N9biPkNYP/W/UvAaUn2aufu1dofAXYfZfhvAAuTPLu9/l3ghnHqeXmSXdvxHsCvAveOMsc84P6qKuAMesFiNFurbVraisn9SU5ptf5SWx2aqHnAfe146Yj3XpFkz/b47CTg5iT709vkfQnwZ/Qee30d2D/JC1sNuyQ5uKoeBDYmOaqN99opXKIkSRO2rYLM7fT+cvsmvcc4N4/dHYC3tc2jd9Jb1fgCvUcRL06yht4nhb4NUFWrgfcBN7ZNpO9vY3wSeEfbULtgeOCq2kDvUdLft7F+Cnx0nHqOAG5v9XwF+OuqugO4A5jTNru+md7jqTckWU0v7Px0tMGq6t+AleltNt7qZt8peg1wdqvhLuC3+t77o/zix68PGHHuucD7216gkSHsa8BVwGrgk+3TVofR2y+zCngH8D+q6qfAqcCft/t1B0/uEzoT+EjrP4hPskmSZpH0FhY02yV5A3BIVf3BTM89d/7Cmn/G1rf1+K9fS9Lsk2RlVS0er5+/2VeSJHXWdDb76imkqj426BokSZosV2QkSVJnGWQkSVJn+WhJA3fo/vMYckOvJGkKXJGRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdtdOgC5DW3LeeBctWjNln3fIlM1SNJKlLXJGRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZAZoCQnJ6kkzxtwHS9N8uK+1+ckedso/RYkWTvGOL+RZGWSNe3rcdurZkmSwCAzaKcDX25fB+mlwIvH6zQBDwL/uaoOBc4ALtkGY0qStFUGmQFJ8gzgaOD1wGv62t/eVjRWJ1ne2g5M8sXWdnuS56Tn/UnWtv6ntb4vTfK5vvHOT7K0Ha9L8u42xpokz0uyADgb+L+SrEry6yPqPLzNuxp441jXVFV3VNUP2su7gF2TzN3K9Z+VZCjJ0BMb1k/izkmS9CR/s+/gnARcXVXfSvLDJIcDz2ztR1bVhiR7tb6XAsur6sokT6MXQH8bWAQcBuwDfC3JjROY98GqekGS3wPeVlVvSPJh4NGq+p8ASV7e1/9C4E1VdWOS90/i+l4F3F5VPx3tzaq6ALgAYO78hTWJcSVJ+jlXZAbndOCydnxZe308cGFVbQCoqoeS7A7sX1VXtraftPePBj5ZVU9U1b8BNwBHTGDev29fVwILxuqYZE9gz6oaDkgTelSU5GDgXOD/nEh/SZKmyhWZAWgrLccBhyYpYA5QwBXbYPjN/GJAfdqI94dXSJ5gO3z/k/wKcCXwuqr6zrYeX5Kkfq7IDMapwCVV9ayqWlBVBwDfBdYDZybZDXqBp6oeAb6f5OTWNre9fxNwWpI5SfYFjgFuA+4BDmr99gRevuX0W3gE2H1kY1U9DDyc5OjW9NqxBmnzrQCWVdXNE5hXkqRpMcgMxun0Vi36fRqYD3wGGEqyChj+CPR/Ad6c5E7gK8B/aOffCawGvgT8cVX9a1V9D/g7YG37escE6vkscMpom32BM4EPtnoyzjhvAg4E/qSNtSrJMycwvyRJU5Iq91lqsObOX1jzzzhvzD7rli+ZoWokSTuCJCuravF4/VyRkSRJneVmX01aklfS+1RSv+9W1SmDqEeSNHsZZDRpVXUNcM2g65AkySCjgTt0/3kMuQdGkjQF7pGRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdtdOgC5DW3LeeBctWTOqcdcuXbKdqJEld4oqMJEnqLIOMJEnqLIOMJEnqLIOMJEnqLIOMJEnqLIOMJEnqLIPMgCV5IsmqJGuTXJFkt2mMtTTJ+dM4d7++19cnWTzZOdr7D7RrWpXkDVOpR5KkiTDIDN7GqlpUVYcAm4Cz+99Mz0x8n5YC+43XaYIub9e0qKo+to3GlCRpCwaZHctNwIFJFiS5O8nFwFrggCSnJ1nTVm7OHT4hyZlJvpXkNuAlfe0XJTm17/Wjfcdvb2OtTrK89VsMXNpWUXbtL2prc0xHkrOSDCUZemLD+m0xpCRpFjLI7CCS7AScAKxpTQuBD1XVwcDjwLnAccAi4IgkJyeZD7ybXrg4GjhoAvOcAJwEHFlVhwHvq6pPAUPAa9sqysa+/pOeA3hVkjuTfCrJAaN1qKoLqmpxVS2es9u8CQwpSdKWDDKDt2uSVfSCxL3Ax1v7PVV1azs+Ari+qh6oqs3ApcAxwJF97ZuAyycw3/HAhVW1AaCqHhqn/2Tn+CywoKqeD1wL/M0EapIkaUr8t5YGb2NVLepvSALw2DTH3UwLqm2PzS7THG9CquqHfS8/BrxvJuaVJM1Orsh0w23AsUn2STIHOB24Afhqa987yc7Aq/vOWQcc3o5PBHZux9cCZw5/OirJXq39EWD3UeYea44ttEdRw04EvjGxS5QkafJckemAqro/yTLgOiDAiqq6CiDJOcAtwMPAqr7TPgpclWQ1cDVthaeqrk6yCBhKsgn4PPAO4CLgw0k2AkeNmHtrc4zmzUlOpLci9BC9T0NJkrRdpKoGXYNmubnzF9b8M86b1Dnrli/ZTtVIknYESVZW1Ra/z2wkHy1JkqTO8tGSpiTJO9lyv8wVVfXeQdQjSZqdDDKakhZYDC2SpIEyyGjgDt1/HkPueZEkTYF7ZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmftNOgCpDX3rWfBshWTOmfd8iXbqRpJUpe4IiNJkjrLICNJkjrLICNJkjrLICNJkjrLICNJkjrLICNJkjrLIDNgSZ5IsirJ2iRXJNltGmMtTXL+NM7dr+/19UkWT3WOJK9KUqONIUnStmKQGbyNVbWoqg4BNgFn97+Znpn4Pi0F9huv00Qk2R14C/DVbTGeJElbY5DZsdwEHJhkQZK7k1wMrAUOSHJ6kjVt5ebc4ROSnJnkW0luA17S135RklP7Xj/ad/z2NtbqJMtbv8XApW11aNf+orY2xxjeA5wL/GRqt0GSpIkxyOwgkuwEnACsaU0LgQ9V1cHA4/SCwXHAIuCIJCcnmQ+8m164OBo4aALznACcBBxZVYcB76uqTwFDwGvb6tDGvv6TmiPJC4ADqmrMX9Wb5KwkQ0mGntiwfryyJUkalUFm8HZNsopekLgX+Hhrv6eqbm3HRwDXV9UDVbUZuBQ4Bjiyr30TcPkE5jseuLCqNgBU1UPj9J/wHO0R2J8DfzheEVV1QVUtrqrFc3abN4GyJUnakv/W0uBtrKpF/Q1JAB6b5ribaUG1BYxdpjneROwOHAJc367hPwCfSXJiVQ3NwPySpFnGFZluuA04Nsk+SeYApwM30NtMe2ySvZPsDLy675x1wOHt+ERg53Z8LXDm8KejkuzV2h+hF0RGGmuOX1BV66tqn6paUFULgFsBQ4wkabtxRaYDqur+JMuA64AAK6rqKoAk5wC3AA8Dq/pO+yhwVZLVwNW0FZ6qujrJImAoySbg88A7gIuADyfZCBw1Yu6tzSFJ0kClqgZdg2a5ufMX1vwzzpvUOeuWL9lO1UiSdgRJVlbVuL+LzEdLkiSps3y0pClJ8k623C9zRVW9dxD1SJJmJ4OMpqQFFkOLJGmgDDIauEP3n8eQe14kSVPgHhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZOw26AGnNfetZsGzFpM9bt3zJdqhGktQlrshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMsgMWJInkqxKsjbJFUl2m8ZYS5OcP41z9+t7fX2SxZOdI8nZSda0a/pykoOmUo8kSRNhkBm8jVW1qKoOATYBZ/e/mZ6Z+D4tBfYbr9ME/G1VHVpVi4D3AX++DcaUJGlUBpkdy03AgUkWJLk7ycXAWuCAJKe3lY61Sc4dPiHJmUm+leQ24CV97RclObXv9aN9x29vY61Osrz1Wwxc2lZSdu0vamtzjKaqftz38ulATelOSJI0Af5m3x1Ekp2AE4CrW9NC4IyqurU98jkXOBz4EfCFJCcDXwXe3drXA9cBd4wzzwnAScCRVbUhyV5V9VCSNwFvq6qh1m+4//wpzPFG4K3ALsBxW+lzFnAWwJw99h1rOEmStsoVmcHbNckqYAi4F/h4a7+nqm5tx0cA11fVA1W1GbgUOAY4sq99E3D5BOY7HriwqjYAVNVD4/Sf9BxV9cGqeg7wduBdW+lzQVUtrqrFc3abN4GyJUnakisyg7ex7Sf5ubYa8tg0x91MC6ptj80u0xxvKi4D/noA80qSZglXZLrhNuDYJPskmQOcDtxA79HSsUn2TrIz8Oq+c9bRexwEcCKwczu+Fjhz+NNRSfZq7Y8Au48y91hzbCHJwr6XS4BvT+wSJUmaPFdkOqCq7k+yjN7+lAArquoqgCTnALcADwOr+k77KHBVktX09t081sa6OskiYCjJJuDzwDuAi4APJ9kIHDVi7q3NMZo3JTkeeJzefp4zpn7lkiSNLVV+qESDNXf+wpp/xnmTPm/d8iXboRpJ0o4gycqq2uL3mY3koyVJktRZPlrSlCR5J1vul7miqt47iHokSbOTQUZT0gKLoUWSNFAGGQ3cofvPY8j9LpKkKXCPjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6iyDjCRJ6qydBl2AtOa+9SxYtmKbjrlu+ZJtOp4kacfkiowkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg8yAJXkiyaoka5NckWS3aYy1NMn50zh3v77X1ydZPNk5khyT5PYkm5OcOpVaJEmaKIPM4G2sqkVVdQiwCTi7/830zMT3aSmw33idJuDeNtbfboOxJEkak0Fmx3ITcGCSBUnuTnIxsBY4IMnpSda0lZtzh09IcmaSbyW5DXhJX/tF/SsiSR7tO357G2t1kuWt32Lg0rY6tGt/UVubYzRVta6q7gR+Nr1bIUnS+PzNvjuIJDsBJwBXt6aFwBlVdWt75HMucDjwI+ALSU4Gvgq8u7WvB64D7hhnnhOAk4Ajq2pDkr2q6qEkbwLeVlVDrd9w//mTnWOC13sWcBbAnD32ne5wkqRZyhWZwds1ySpgiN5jmY+39nuq6tZ2fARwfVU9UFWbgUuBY4Aj+9o3AZdPYL7jgQuragNAVT00Tv+pzDGuqrqgqhZX1eI5u83bFkNKkmYhV2QGb2NVLepvaKshj01z3M20oNr22OwyzfEkSdrhuCLTDbcBxybZJ8kc4HTgBnqPlo5NsneSnYFX952zjt7jIIATgZ3b8bXAmcOfjkqyV2t/BNh9lLnHmkOSpIEyyHRAVd0PLKO3P2U1sLKqrmrt5wC3ADcD3+g77aP0Ashq4CjaCk9VXQ18Bhhqj7Te1vpfBHx45GbfcebYQpIjknyfXuD5SJK7pn7lkiSNLVU16Bo0y82dv7Dmn3HeNh1z3fIl23Q8SdLMSrKyqrb4fWYjuSIjSZI6y82+mpIk72TL/TJXVNV7B1GPJGl2MshoSlpgMbRIkgbKIKOBO3T/eQy5p0WSNAXukZEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ2106ALkNbct54Fy1YMuowdxrrlSwZdgiR1hisykiSpswwykiSpswwykiSpswwykiSpswwykiSpswwykiSpswwyA5DkiSSrkqxNckWS3bbRuF/ZSvtFSU6d5FjnJHnbKO0Lkqwd47wXtmtblWR1klMmM68kSZNhkBmMjVW1qKoOATYBZ/e/mZ5Jf2+q6sXborgk0/n9QmuBxVW1CPhN4CPTHE+SpK0yyAzeTcCBbaXj7iQX0wsDByQ5PcmatnJzLkCSs5O8f/jkJEuTnN+OH21fk+T8Nt4XgWf29T88yQ1JVia5Jsn81n59kvOSDAFv6S+wnbM6yWrgjWNdTFVtqKrN7eXTgJre7ZEkaesMMgPUVipOANa0poXAh6rqYOBx4FzgOGARcESSk4FPA/2Pa04DLhsx9CnAc4GDgNcBL27z7Qz8FXBqVR0OfAJ4b995u1TV4qr6sxHjXQj8flUdNsHrOjLJXe26zu4LNv19zkoylGToiQ3rJzKsJElbMMgMxq5JVgFDwL3Ax1v7PVV1azs+Ari+qh5oQeBS4JiqegD4lyQvSrI38Dzg5hHjHwN8sqqeqKofAF9q7c8FDgGubfO/C/iVvvMuH1lokj2BPavqxtZ0yXgXV1VfbWHsCOD/TvK0Ufpc0ELT4jm7zRtvSEmSRuXehcHY2PaQ/FwSgMcmeP5lwO8A3wSurKqJPr4JcFdVHbWV9yc6/4RU1Tfa465D6IU2SZK2KVdkdly3Accm2SfJHOB04Ib23pXASa1t5GMlgBuB05LMaXtgXtba7wb2TXIU9B41JTl4rCKq6mHg4SRHt6bXjtU/ya8Ob+5N8ix6K0brxrxSSZKmyBWZHVRV3Z9kGXAdvZWUFVV1VXvvR0m+ARxUVbeNcvqV9PbWfJ3eo6tb2nmb2sewP5BkHr3v/3nAXeOUcybwiSQFfGGcvkcDy5I8DvwM+L2qenD8K5YkafIy8acS0vYxd/7Cmn/GeYMuY4exbvmSQZcgSQOXZGVVLR6vn4+WJElSZ/loSVOS5JX0Ph59ry5SAAAJ7ElEQVTe77tV5W/ylSTNGIOMpqSqrgGuGXQdkqTZzSCjgTt0/3kMuS9EkjQF7pGRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdZZCRJEmdtdOgC5DW3LeeBctWDLoMTcK65UsGXYIkAa7ISJKkDjPISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLIDECSJ5KsSrI2yRVJdttG435lK+0XJTl1kmOdk+Rto7QvSLJ2jPP2TnJdkkeTnD+ZOSVJmiyDzGBsrKpFVXUIsAk4u//N9Ez6e1NVL94WxSWZzu8X+gnw/wBbhCBJkrY1g8zg3QQc2FY67k5yMbAWOCDJ6UnWtJWbcwGSnJ3k/cMnJ1k6vPKR5NH2NUnOb+N9EXhmX//Dk9yQZGWSa5LMb+3XJzkvyRDwlv4C2zmrk6wG3jjWxVTVY1X1ZXqBRpKk7cogM0Bt5eMEYE1rWgh8qKoOBh4HzgWOAxYBRyQ5Gfg0cErfMKcBl40Y+hTgucBBwOuAF7f5dgb+Cji1qg4HPgG8t++8XapqcVX92YjxLgR+v6oOm8bl/oIkZyUZSjL0xIb122pYSdIsY5AZjF2TrAKGgHuBj7f2e6rq1nZ8BHB9VT1QVZuBS4FjquoB4F+SvCjJ3sDzgJtHjH8M8MmqeqKqfgB8qbU/FzgEuLbN/y7gV/rOu3xkoUn2BPasqhtb0yVTv+wnVdUFLTQtnrPbvG0xpCRpFvLfWhqMjVW1qL8hCcBjEzz/MuB3gG8CV1ZVTfC8AHdV1VFbeX+i80uStENwRWbHdRtwbJJ9kswBTgduaO9dCZzU2kY+VgK4ETgtyZy2B+Zlrf1uYN8kR0HvUVOSg8cqoqoeBh5OcnRreu10LkqSpG3JFZkdVFXdn2QZcB29lZQVVXVVe+9HSb4BHFRVt41y+pX09tZ8nd6jq1vaeZvax7A/kGQeve//ecBd45RzJvCJJAV8Ybzak6wD9gB2aft6XlFVXx/vPEmSJisTfyohbR9z5y+s+WecN+gyNAnrli8ZdAmSnuKSrKyqxeP189GSJEnqLB8taUqSvJLex8P7fbeqThmtvyRJ24NBRlNSVdcA1wy6DknS7GaQ0cAduv88htxzIUmaAvfISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkztpp0AVIa+5bz4JlKwZdhiRpmtYtXzLjc7oiI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgM0sluS7JK0e0/UGSvx7jnEe3f2WSJE2cQWb2+iTwmhFtr2ntkiR1gkFm9voUsCTJLgBJFgD7AXck+acktydZk+SkkScmeWmSz/W9Pj/J0nZ8eJIbkqxMck2S+TNxMZKk2ckgM0tV1UPAbcAJrek1wN8BG4FTquoFwMuAP0uSiYyZZGfgr4BTq+pw4BPAe7d17ZIkDfM3+85uw4+XrmpfXw8E+B9JjgF+BuwP/DLwrxMY77nAIcC1LfvMAe4frWOSs4CzAObsse+0LkKSNHsZZGa3q4C/SPICYLeqWtkeEe0LHF5VjydZBzxtxHmb+cXVvOH3A9xVVUeNN3FVXQBcADB3/sKa1lVIkmYtHy3NYlX1KHAdvUdAw5t85wH/3kLMy4BnjXLqPcBBSeYm2RN4eWu/G9g3yVHQe9SU5ODtehGSpFnNFRl9EriSJz/BdCnw2SRrgCHgmyNPqKrvJfk7YC3wXeCO1r4pyanAB5LMo/fzdR5w13a/CknSrGSQmeWq6h/oPRIafv0gMOqjoap6Rt/xHwN/PEqfVcAx275SSZK25KMlSZLUWQYZSZLUWQYZSZLUWQYZSZLUWW721cAduv88hpYvGXQZkqQOckVGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1Vqpq0DVolkvyCHD3oOvYAewDPDjoInYA3oce74P3YNhsvQ/Pqqp9x+vkP1GgHcHdVbV40EUMWpIh74P3YZj3wXswzPswNh8tSZKkzjLISJKkzjLIaEdwwaAL2EF4H3q8Dz3eB+/BMO/DGNzsK0mSOssVGUmS1FkGGc2YJL+Z5O4k/5xk2Sjvz01yeXv/q0kWzHyV298E7sMxSW5PsjnJqYOocXubwD14a5KvJ7kzyT8ledYg6tzeJnAfzk6yJsmqJF9OctAg6tzexrsPff1elaSSPCU/wTOBn4elSR5oPw+rkrxhEHXucKrKP/7Z7n+AOcB3gGcDuwCrgYNG9Pk94MPt+DXA5YOue0D3YQHwfOBi4NRB1zyge/AyYLd2/N9n8c/CHn3HJwJXD7ruQdyH1m934EbgVmDxoOse0M/DUuD8Qde6o/1xRUYz5YXAP1fVv1TVJuAy4KQRfU4C/qYdfwp4eZLMYI0zYdz7UFXrqupO4GeDKHAGTOQeXFdVG9rLW4FfmeEaZ8JE7sOP+14+HXgqbmqcyP82ALwHOBf4yUwWN4Mmeh80gkFGM2V/4Ht9r7/f2kbtU1WbgfXA3jNS3cyZyH14qpvsPXg98I/btaLBmNB9SPLGJN8B3ge8eYZqm0nj3ockLwAOqKoVM1nYDJvofxevao9cP5XkgJkpbcdmkJG0w0ryu8Bi4P2DrmVQquqDVfUc4O3AuwZdz0xL8kvAnwN/OOhadgCfBRZU1fOBa3lyBXtWM8hoptwH9P+/h19pbaP2SbITMA/44YxUN3Mmch+e6iZ0D5IcD7wTOLGqfjpDtc2kyf4sXAacvF0rGozx7sPuwCHA9UnWAS8CPvMU3PA77s9DVf2w77+FjwGHz1BtOzSDjGbK14CFSX41yS70NvN+ZkSfzwBntONTgS9V2+H2FDKR+/BUN+49SPIfgY/QCzH/PoAaZ8JE7sPCvpdLgG/PYH0zZcz7UFXrq2qfqlpQVQvo7Zk6saqGBlPudjORn4f5fS9PBL4xg/XtsPxHIzUjqmpzkjcB19Dbnf+Jqroryf8LDFXVZ4CPA5ck+WfgIXr/IT+lTOQ+JDkCuBL434D/nOTdVXXwAMvepib4s/B+4BnAFW2/971VdeLAit4OJngf3tRWph4HfsSTQf8pY4L34SlvgvfhzUlOBDbT+9/IpQMreAfib/aVJEmd5aMlSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWf8/HC0QiNkTDdkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plot graph of feature importances for better visualization\n",
    "plt.figure(figsize=(7,7))\n",
    "feat_importances = pd.Series(ml_rfe.feature_importances_, index=train.columns)\n",
    "feat_importances.nlargest(10).plot(kind='barh')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recursive Feature Elimination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we are going to choose the top 20 features and use those ones to work with.\n",
    "from sklearn.feature_selection import RFE\n",
    "select = RFE(ml_rfe,n_features_to_select=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RFE(estimator=RandomForestClassifier(bootstrap=True, class_weight=None,\n",
       "                                     criterion='gini', max_depth=10,\n",
       "                                     max_features=0.45510204081632655,\n",
       "                                     max_leaf_nodes=None,\n",
       "                                     min_impurity_decrease=0.0,\n",
       "                                     min_impurity_split=None,\n",
       "                                     min_samples_leaf=2, min_samples_split=2,\n",
       "                                     min_weight_fraction_leaf=0.0,\n",
       "                                     n_estimators=49, n_jobs=-1,\n",
       "                                     oob_score=False, random_state=None,\n",
       "                                     verbose=0, warm_start=False),\n",
       "    n_features_to_select=10, step=1, verbose=0)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we fit the new model based on the top 10 best features\n",
    "select.fit(X_train_sample, y_train_sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's get those features and use them in the model that we worked with before and see how it responds\n",
    "X_train_rfe = select.transform(X_train_sample)\n",
    "X_test_rfe = select.transform(X_test_sample)\n",
    "\n",
    "X_valid_rfe= select.transform(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_score_(m,md):\n",
    "    \n",
    "    # Used to ignore warnings generated from StackingCVClassifier\n",
    "    import warnings\n",
    "    warnings.simplefilter('ignore')\n",
    "\n",
    "    #Call the classification metrics we want to use\n",
    "    from sklearn.metrics import roc_auc_score,auc,f1_score,roc_curve\n",
    "    \n",
    "    \n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test_sample, m)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "    \n",
    "    f1 = f1_score(y_test_sample,m)\n",
    "    roc = roc_auc_score(y_test_sample,m)\n",
    "    #oob = md.oob_score_ #I added the oob score\n",
    "    \n",
    "    #let's see how the model does on validation data\n",
    "    pred = md.predict(X_valid_rfe)\n",
    "    f2 = f1_score(y_valid,pred)\n",
    "    \n",
    "    pred_3 = md.predict(X_train_rfe)\n",
    "    f3 = f1_score(y_train_sample,pred_3)\n",
    "    \n",
    "    res = [\"roc_auc-->\",roc_auc,\"F1_score-->\",f1,\"roc_auc_score-->\",roc]\n",
    "    print(res)\n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on out of sample data\")\n",
    "    \n",
    "    valid = ['Valid_data_F1_score-->',f2]\n",
    "    \n",
    "    print(valid)\n",
    "    \n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on train data\")\n",
    "    \n",
    "    train_ = ['Train_data_F1_score-->',f3]\n",
    "    \n",
    "    print(train_)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('--'*10)\n",
    "    print('Scores on the Validation Data according to the Confusion_matrix')\n",
    "    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    tn, fp, fn, tp = confusion_matrix(y_valid,pred).ravel()\n",
    "    \n",
    "    # Error rate : \n",
    "    err_rate = (fp + fn) / (tp + tn + fn + fp)\n",
    "    print(\"Error rate  : \", err_rate)\n",
    "\n",
    "    # Accuracy : \n",
    "    acc_ = (tp + tn) / (tp + tn + fn + fp)\n",
    "    print(\"Accuracy  : \", acc_)\n",
    "    \n",
    "    \n",
    "    #precision\n",
    "    preci_ = tp / (tp + fp)\n",
    "    print(\"Precision  : \", preci_)\n",
    "\n",
    "    # Sensitivity : \n",
    "    sens_ = tp / (tp + fn)\n",
    "    print(\"Recall  : \", sens_)\n",
    "\n",
    "    # Specificity \n",
    "    sp_ = tn / (tn + fp)\n",
    "    print(\"Specificity  : \", sens_)\n",
    "\n",
    "    # False positive rate (FPR)\n",
    "    FPR = fp / (tn + fp)\n",
    "    print(\"False positive rate  : \", FPR)\n",
    "    \n",
    "    # False positive rate (FPR)\n",
    "    TPR = tp / (tp + fn)\n",
    "    print(\"True positive rate  : \", TPR)\n",
    "    \n",
    "    print('--'*10)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('Drawing the Confusion Matrix on Test_data')\n",
    "    #from sklearn.metrics import confusion_matrix\n",
    "    \n",
    "    plt.figure(figsize=(7,7))\n",
    "    cm_2 = confusion_matrix(y_test_sample,m)\n",
    "    sns.heatmap(cm_2,annot=True,fmt=\"d\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['roc_auc-->', 0.8805222591948372, 'F1_score-->', 0.64, 'roc_auc_score-->', 0.8805222591948372]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.5380116959064327]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.9965283018867924]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.00395\n",
      "Accuracy  :  0.99605\n",
      "Precision  :  0.4380952380952381\n",
      "Recall  :  0.696969696969697\n",
      "Specificity  :  0.696969696969697\n",
      "False positive rate  :  0.002959767231865155\n",
      "True positive rate  :  0.696969696969697\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGfNJREFUeJzt3X201WWd9/H3V4jyoREQJEVzUNGynBhDSycfRlPQUnTGKV1Ncitz4yQ1ZbNKHZubfFjdeqvVNPlwY6KoBXGLKTmiIj6XD7hGdEQjT/jAOT4gIlpZnfBc9x/nBx0Z4cBxc/aX83u/XL/l3tf+7b2vvXSd7/p+fte+dpRSkCSp2TZr9gQkSQILkiQpCQuSJCkFC5IkKQULkiQpBQuSJCkFC5IkKQULkiQpBQuSJNVQREyNiKUR8XiXsW9GRFtELKiOI7o8dkZEtETEoogY02V8bDXWEhGndxkfEREPVuM/jogB3c5pY+/U8Mdli90KQr1m8+33b/YUVDMr29uiUa/VyL+X7xqy8zrnFREHAL8Bri6lfLga+ybwm1LKhWucuwcwHdgH2B64HditeviXwKFAKzAfOL6U8kREzASuL6XMiIjLgEdLKZeua052SJJUQ6WUe4Dl63n6OGBGKeUPpZSngRY6i9M+QEspZXEppR2YAYyLiAAOBq6rnj8NOLq7N7EgSVIWHW827ui5L0bEY1WkN6gaGw4s6XJOazW2tvFtgBWllJVrjK+TBUmS+qCImBgRD3c5Jq7H0y4FdgFGAS8AF23USa6hf2++mSRpHUpH416qlCnAlA18zkurbkfE5cBN1d02YMcup+5QjbGW8VeAgRHRv+qSup6/VnZIkpRFR0fjjh6IiO263D0GWLUCbzZwXES8OyJGACOBh+hcxDCyWlE3ADgOmF06V8vdCRxbPX88cGN372+HJEk1FBHTgYOAIRHRCkwGDoqIUUABngFOBiilLKxWzT0BrAQmlVLerF7ni8CtQD9gaillYfUWpwEzIuJc4BHgim7n5LJv9SUu+1Zva+Sy7/bnFzbs7+WA7T/UsHn1FjskScqih1FbX+E1JElSCnZIkpRFA1fZbYosSJKUxTv7Qusmz8hOkpSCHZIkZWFkJ0lKwVV2kiQ1nx2SJCVRjOwkSSkY2UmS1Hx2SJKUhZGdJCkFvxgrSVLz2SFJUhZGdpKkFFxlJ0lS89khSVIWRnaSpBSM7CRJaj47JElKopR6fw/JgiRJWdT8GpKRnSQpBTskScqi5osaLEiSlEXNIzsLkiRl4eaqkiQ1nx2SJGVhZCdJSqHmixqM7CRJKdghSVIWRnaSpBSM7CRJaj47JEnKouYdkgVJkpKo+27fRnaSpBTskCQpCyM7SVIKNV/2bWQnSUrBDkmSsjCykySlYGQnSVLz2SFJUhZGdpKkFIzsJElqPjskScrCyE6SlELNC5KRnSQpBTskScqi5osaLEiSlIWRnSRJzWeHJElZGNlJklIwspMkqfnskCQpCyM7SVIKRnaSJDWfHZIkZVHzDsmCJElZlNLsGTSVkZ0kKQU7JEnKwshOkpRCzQuSkZ0kKQU7JEnKwi/GSpJSMLKTJKn57JAkKQu/hyRJSqGjo3FHNyJiakQsjYjHu4xdEBG/iIjHIuInETGwy2NnRERLRCyKiDFdxsdWYy0RcXqX8RER8WA1/uOIGNDdnCxIklRPVwFj1xibC3y4lPIXwC+BMwAiYg/gOOBD1XMuiYh+EdEPuBg4HNgDOL46F+B84DullF2BV4EJ3U3IgiRJWfRih1RKuQdYvsbYbaWUldXdB4AdqtvjgBmllD+UUp4GWoB9qqOllLK4lNIOzADGRUQABwPXVc+fBhzd3ZwsSJKURelo2BEREyPi4S7HxA2czUnAnOr2cGBJl8daq7G1jW8DrOhS3FaNr5OLGiSpDyqlTAGm9OS5EXEmsBL4YUMn1Q0LkiQlUTqav8ouIv4H8GngkFJWL/trA3bsctoO1RhrGX8FGBgR/asuqev5a2VkJ0lZ9OI1pLcTEWOBrwNHlVLe6PLQbOC4iHh3RIwARgIPAfOBkdWKugF0LnyYXRWyO4Fjq+ePB27s7v0tSJJUQxExHbgf2D0iWiNiAvB94L3A3IhYEBGXAZRSFgIzgSeAW4BJpZQ3q+7ni8CtwJPAzOpcgNOAr0ZEC53XlK7odk5lI38R64/LFje/B1VtbL79/s2egmpmZXtbNOq13rj0Sw37e7nFF/69YfPqLV5DkqQsElxDaiYjO0lSCnZIkpRFzXf7tiBJUhYWJElSCu72LUlS89khSVIWRnbamL7xrW9zz88eYvCggdxw7WUAXHzFtcyafQuDBm4NwJdPHs8B++0DwOVX/5jrb7qVfpttxhmnfoG/+thHAbhm5g3Mmn0LpRSOPWosn//sMQD84qnFnHPBv/PG737P9ttty/mTv85WW27ZhE+qTcXlUy7iU0d8kqUvL2PUXx4CwFnf/BpHHnkYHR2Fl5cu46R/OJUXXnipyTOtIZd9a2M6+ohDuezb5/638c9/9mhmTbuYWdMuXl2MfvX0s8yZdzc3XnsZl337XM658Pu8+eabPLX4GWbNvoXpP/gus6Zdwt0/f4jnWp8HYPJ53+UrXziRn1xzKYccsB9X/nBWr34+bXquvnomn/r0594yduFFl7LXRw9l9N6H8R833843zjy1SbNTnXVbkCLiAxFxWkR8rzpOi4gP9sbk+oLRo/Zk6z9773qde8e9D3D4IQcyYMAAdtj+fbx/h+35ryd/yeJnlrDnh3Zn8/e8h/79+zF61J7cfvfPAHh2SRujR+0JwL5778Xcu+/baJ9FfcO99z3I8ldXvGXs17/+zerbW265BRt7BxetRQN/fmJTtM6CFBGn0fmDS0HnRnoPVbend/2pWm246bN+yjEnfIFvfOvbvPb6rwFY+vIrvG/Y0NXnDNt2CEtfXsauO+/Efz66kBWvvc7vfv977r1/Pi++9DIAu4zYiTvuvR+A2+68lxdfWtb7H0Z9wjlnn8bTv5rP8ccfwzfPuqDZ06mnjtK4YxPUXYc0Adi7lHJeKeXa6jiPzl8JXOvP0Xb9YagfXD29kfPtEz57zKeYM3Mqs666mKHbDOaC71++zvN3+fP3c9Ln/o6Jp57JP371X9l95M5stlnnf7pz/uVUZlx/E5856Uv89o3f8a53eVlQPfOv/+t8RuyyN9On/4RJp5zY7Omohrr769UBbA88u8b4dtVjb6vrD0O5uep/N2TwoNW3jz3qcCZ9bTIA2w7dZnXnA/DS0mVsO3QIAH975Bj+9sgxAHz3sqt437ad4zvvtCOXf/dbADzzXCv3/PyhXvkM6rt+NP16fjr7Gs46+6JmT6V2Ss1X2XXXIX0FmBcRcyJiSnXcAswDvrzxp9c3vbzsTz9jP+/un7PrzjsB8Nef+Dhz5t1Ne3s7rc+/yHOtz7PnB3cD4JUq83/hxaXMu/tnHHHoQW8Z7+jo4P9Om8Fnjj6iFz+J+opddx2x+vZRR45h0aJfNXE2NVbzyG6dHVIp5ZaI2I3OiG7V76G3AfNLKW9u7Mn1BV+bfB7zH3mMFSte55Cj/55TJnye+Y88xqKnFkPA8PcNY/LX/wmAXXfeiTEH789RnzuZ/v36ceZXT6Ffv34AnPov57Li9dfp378/Z/7zKfzZe7cC4Oa5dzHj+psA+OSB+3HMpw5rzgfVJuPaay7mwAP2ZciQwTyz+GHOOvtCDj/8YHbbbRc6Ojp47rk2TpnkJWL1Pn8PSX2Kv4ek3tbI30P67bl/37C/l1t+41p/D0mS1EObaNTWKH4xVpKUgh2SJGVR81V2FiRJysLITpKk5rNDkqQsNtE96BrFgiRJWRjZSZLUfHZIkpRE3feysyBJUhZGdpIkNZ8dkiRlUfMOyYIkSVnUfNm3kZ0kKQU7JEnKwshOkpRBqXlBMrKTJKVghyRJWdS8Q7IgSVIWNd+pwchOkpSCHZIkZWFkJ0lKoeYFychOkpSCHZIkJVFKvTskC5IkZWFkJ0lS89khSVIWNe+QLEiSlIR72UmSlIAdkiRlUfMOyYIkSVnUeys7IztJUg52SJKURN0XNViQJCmLmhckIztJUgp2SJKURc0XNViQJCmJul9DMrKTJKVghyRJWRjZSZIyMLKTJCkBOyRJysLITpKUQbEgSZJSqHlB8hqSJCkFOyRJSsLITpKUQ80LkpGdJCkFOyRJSsLITpKUQt0LkpGdJNVURHw5Ih6PiIUR8ZVqbHBEzI2Ip6p/D6rGIyK+FxEtEfFYROzV5XXGV+c/FRHjezofC5IkJVE6Gnd0JyI+DPxPYB/gI8CnI2JX4HRgXillJDCvug9wODCyOiYCl1avMxiYDHyseq3Jq4rYhrIgSVIWJRp3dO+DwIOllDdKKSuBu4G/AcYB06pzpgFHV7fHAVeXTg8AAyNiO2AMMLeUsryU8iowFxjbk49vQZKkenoc2D8itomILYAjgB2BYaWUF6pzXgSGVbeHA0u6PL+1Glvb+AZzUYMkJdHIRQ0RMZHOaG2VKaWUKavfq5QnI+J84Dbgt8AC4M23zKeUEhG99psYFiRJSqJ0rFfUtn6v1Vl8pnRzzhXAFQAR8S06u5uXImK7UsoLVSS3tDq9jc4OapUdqrE24KA1xu/qyZyN7CSppiJi2+rf76fz+tGPgNnAqpVy44Ebq9uzgROq1XYfB16ror1bgcMiYlC1mOGwamyD2SFJUhJN+B7SrIjYBvgjMKmUsiIizgNmRsQE4FngM9W5N9N5nakFeAM4EaCUsjwizgHmV+edXUpZ3pPJWJAkKYmyfqvjGvh+Zf+3GXsFOORtxgswaS2vMxWY+k7nY2QnSUrBDkmSkqj71kEWJElKopGr7DZFRnaSpBTskCQpidJrX0HNyYIkSUkY2UmSlIAdkiQlUfcOyYIkSUnU/RqSkZ0kKQU7JElKwshOkpRCb+9ll42RnSQpBTskSUrCvewkSSl0GNlJktR8dkiSlETdFzVYkCQpibov+zaykySlYIckSUnUfesgC5IkJWFkJ0lSAnZIkpRE3b+HZEGSpCTqvuzbyE6SlIIdkiQl4So7SVIKdb+GZGQnSUrBDkmSkqj7ogYLkiQlUfdrSEZ2kqQUNnqHtPn2+2/st5CkPqHuixqM7CQpibpfQzKykySlYIckSUkY2UmSUqj5IjsLkiRlUfcOyWtIkqQU7JAkKYm6r7KzIElSEh3NnkCTGdlJklKwQ5KkJApGdpKkBDpqvu7byE6SlIIdkiQl0WFkJ0nKoO7XkIzsJEkp2CFJUhJ1/x6SBUmSkjCykyQpATskSUrCyE6SlELdC5KRnSQpBTskSUqi7osaLEiSlERHveuRkZ0kKQc7JElKwr3sJEkp1PzXJ4zsJEk52CFJUhJ1/x6SBUmSkuiIel9DMrKTJKVghyRJSdR9UYMFSZKSqPs1JCM7SVIKdkiSlETdtw6yIElSEnXfqcHITpJqKiIGRsR1EfGLiHgyIvaNiMERMTcinqr+Pag6NyLiexHREhGPRcReXV5nfHX+UxExvqfzsSBJUhKlgcd6+jfgllLKB4CPAE8CpwPzSikjgXnVfYDDgZHVMRG4FCAiBgOTgY8B+wCTVxWxDWVBkqQkOqJxR3ciYmvgAOAKgFJKeyllBTAOmFadNg04uro9Dri6dHoAGBgR2wFjgLmllOWllFeBucDYnnx+C5Ik1dMI4GXgyoh4JCJ+EBFbAsNKKS9U57wIDKtuDweWdHl+azW2tvENZkGSpCQ6GnhExMSIeLjLMXGNt+sP7AVcWkr5S+C3/CmeA6CUsoEJ4DvjKjtJSqKRf/lLKVOAKes4pRVoLaU8WN2/js6C9FJEbFdKeaGK5JZWj7cBO3Z5/g7VWBtw0Brjd/VkznZIklRDpZQXgSURsXs1dAjwBDAbWLVSbjxwY3V7NnBCtdru48BrVbR3K3BYRAyqFjMcVo1tMDskSUqiCV+M/RLww4gYACwGTqSzUZkZEROAZ4HPVOfeDBwBtABvVOdSSlkeEecA86vzzi6lLO/JZCxIkpREb+9lV0pZAIx+m4cOeZtzCzBpLa8zFZj6TudjZCdJSsEOSZKSqPtu3xYkSUqi1HsrOyM7SVIOdkiSlISRnSQphboXJCM7SVIKdkiSlESvbRqXlAVJkpKo+0+YG9lJklKwQ5KkJOq+qMGCJElJ1L0gGdlJklKwQ5KkJFxlJ0lKoe6r7CxIkpSE15AkSUrADkmSkvAakiQphY6alyQjO0lSCnZIkpRE3Rc1WJAkKYl6B3ZGdpKkJOyQJCkJIztJUgp136nByE6SlIIdkiQlUffvIVmQJCmJepcjIztJUhJ2SJKUhKvsJEkp1P0akpGdJCkFOyRJSqLe/ZEFSZLSqPs1JCM7SVIKdkiSlETdFzVYkCQpiXqXIyM7SVISdkiSlETdFzVYkCQpiVLz0M7ITpKUgh2SJCVhZCdJSqHuy76N7CRJKdghSVIS9e6PLEiSlIaRndJp+eUDPPKft/Pw/Nt44P6bmz0d9TGXT7mI51sfZcEj894yPumUE3n8v+7m0QV3cN7/PrNJs1Od2SEl9clD/45XXnm12dNQH3T11TO55JIrufLKf1s9dtCB+3HUkWPY66OH0t7eztCh2zRxhvVV91V2dkhSzdx734Msf3XFW8ZOPvkE/s8FF9Pe3g7Ayy+/0oyp1V5p4D+boh4XpIg4sZET0Z+UUphz83QefGAO/zDhc82ejmpg5Mid+cQn9uHn9/2UO26/jtEf/Uizp6QaeieR3VnAlY2aiP7kwL8+hueff5GhQ7fhljkzWLSohXvve7DZ01If1r9/PwYNGsh+nziSvUePYvqPLmPk7vs2e1q1U/fIbp0FKSIeW9tDwLB1PG8iMBEg+m3NZptt2eMJ1tHzz78IdMYmN944h733HmVB0kbV1voCN9wwB4D5Dy+go6ODIUMGs2zZ8ibPrF421aitUbqL7IYBJwBHvs2x1pC5lDKllDK6lDLaYrRhtthic7baasvVtw/95IEsXLioybNSX3fj7Fs56KD9gM74bsCAARYj9bruIrubgK1KKQvWfCAi7tooM6q5YcOGct3/uwLojFFmzLiBW2+7q7mTUp9y7TUXc+AB+zJkyGCeWfwwZ519IVdeNYMfXH4RCx6ZR3v7HzlpwleaPc1aqntkF6Vs3Bax/4Dh9e5BJfVpK9vbolGv9fmd/qZhfy+vefb6hs2rt7jsW5KUgl+MlaQk6h4nWZAkKQn3spMkKQE7JElKou7fQ7IgSVISdV/2bWQnSUrBDkmSkqj7ogYLkiQlUfdrSEZ2kqQU7JAkKYm6L2qwIElSEht7b9HsjOwkqYYi4j0R8VBEPBoRCyPirGp8REQ8GBEtEfHjiBhQjb+7ut9SPf7nXV7rjGp8UUSM6emcLEiSlEQHpWHHevgDcHAp5SPAKGBsRHwcOB/4TillV+BVYEJ1/gTg1Wr8O9V5RMQewHHAh4CxwCUR0a8nn9+CJElJdDTw6E7p9Jvq7ruqowAHA9dV49OAo6vb46r7VI8fEhFRjc8opfyhlPI00ALss6GfHSxIkpRGaeA/ETExIh7uckxc8/0iol9ELACWAnOBXwErSikrq1NageHV7eHAEoDq8deAbbqOv81zNoiLGiSpDyqlTAGmdHPOm8CoiBgI/AT4QG/MbW0sSJKURLN2aiilrIiIO4F9gYER0b/qgnYA2qrT2oAdgdaI6A9sDbzSZXyVrs/ZIEZ2kpREKaVhR3ciYmjVGRERmwOHAk8CdwLHVqeNB26sbs+u7lM9fkfpfKPZwHHVKrwRwEjgoZ58fjskSaqn7YBp1Yq4zYCZpZSbIuIJYEZEnAs8AlxRnX8FcE1EtADL6VxZRyllYUTMBJ4AVgKTqihwg8XG/iJW/wHD6/1NL0l92sr2tmjUa43Z8fCG/b28dcmchs2rt9ghSVISbq4qSVICdkiSlIS/hyRJSsHNVSVJSsAOSZKSMLKTJKXgKjtJkhKwQ5KkJDpqvqjBgiRJSdS7HBnZSZKSsEOSpCRcZSdJSqHuBcnITpKUgh2SJCVR962DLEiSlISRnSRJCdghSVISdd86yIIkSUnU/RqSkZ0kKQU7JElKou6LGixIkpSEkZ0kSQnYIUlSEkZ2kqQU6r7s28hOkpSCHZIkJeEvxkqSUjCykyQpATskSUrCyE6SlIKRnSRJCdghSVISRnaSpBSM7CRJSsAOSZKSMLKTJKVgZCdJUgJ2SJKURCkdzZ5CU1mQJCmJuv8ekpGdJCkFOyRJSqK4yk6SlIGRnSRJCdghSVISRnaSpBTqvlODkZ0kKQU7JElKou5bB1mQJCkJryFJklJw2bckSQnYIUlSEkZ2kqQUXPYtSVICdkiSlISRnSQpBVfZSZKUgB2SJCVhZCdJSsFVdpIkJWCHJElJuLmqJCkFIztJkhKwQ5KkJFxlJ0lKoe7XkIzsJEkpWJAkKYlSSsOO9RERYyNiUUS0RMTpG/njdcvITpKS6M1rSBHRD7gYOBRoBeZHxOxSyhO9Nok12CFJUj3tA7SUUhaXUtqBGcC4Zk7IgiRJSZQGHuthOLCky/3WaqxpNnpkt7K9LTb2e/RFETGxlDKl2fNQffj/XPM18u9lREwEJnYZmpL9v68dUl4Tuz9Faij/n+tDSilTSimjuxxrFqM2YMcu93eoxprGgiRJ9TQfGBkRIyJiAHAcMLuZE3KVnSTVUCllZUR8EbgV6AdMLaUsbOacLEh5pc561Sf5/1zNlFJuBm5u9jxWibrvnSRJysFrSJKkFCxICWXbzkN9V0RMjYilEfF4s+ciWZCS6bKdx+HAHsDxEbFHc2elPuwqYGyzJyGBBSmjdNt5qO8qpdwDLG/2PCSwIGWUbjsPSeoNFiRJUgoWpHzSbechSb3BgpRPuu08JKk3WJCSKaWsBFZt5/EkMLPZ23mo74qI6cD9wO4R0RoRE5o9J9WXOzVIklKwQ5IkpWBBkiSlYEGSJKVgQZIkpWBBkiSlYEGSJKVgQZIkpWBBkiSl8P8B52EefveD3qIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#so I am going to use the model that got us 81% before on the test data and see if we improve\n",
    "ml_rfe = RandomForestClassifier(n_jobs=-1, n_estimators=100, min_samples_leaf=2, max_features=0.45510204081632655, bootstrap=True, max_depth=10, random_state=45)\n",
    "ml_rfe.fit(X_train_rfe, y_train_sample)\n",
    "pred_d = ml_rfe.predict(X_test_rfe)\n",
    "print_score_(pred_d,ml_rfe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid Search based CV\n",
    "\n",
    "Now that we have some information on the parameter ranges, lets  use the grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 50, stop = 10000, num = 100)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto','log2','sqrt']#[float(x) for x in np.linspace(start = 0.4, stop = 0.85, num = 20)]\n",
    "# Maximum number of levels in tree\n",
    "#max_depth = [int(x) for x in np.linspace(1, 10, 10)]\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_leaf = [int(x) for x in np.linspace(20, 300, 10)]\n",
    "# Method of selecting samples for training each tree\n",
    "#bootstrap = [True]\n",
    "\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               #'max_depth': [8],\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               #'bootstrap': bootstrap\n",
    "              }\n",
    "\n",
    "#pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done  45 out of  50 | elapsed:  4.8min remaining:   32.3s\n",
      "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed:  5.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done  45 out of  50 | elapsed:  5.1min remaining:   33.9s\n",
      "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed:  5.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:  1.3min\n"
     ]
    }
   ],
   "source": [
    "rm_clf = RandomForestClassifier()\n",
    "\n",
    "ls = []\n",
    "max_iterations = 5\n",
    "i = 0\n",
    "\n",
    "\n",
    "while i<max_iterations:\n",
    "    # Random search of parameters, using 3 fold cross validation, \n",
    "    # search across 100 different combinations, and use all available cores\n",
    "    grid = RandomizedSearchCV(rm_clf, param_distributions=random_grid,\n",
    "                              n_iter = 10, \n",
    "                              scoring='f1', \n",
    "                              cv =KFold(n_splits=5, shuffle=True), \n",
    "                              verbose=2, \n",
    "                              #random_state=42, \n",
    "                              n_jobs=-1,\n",
    "                              return_train_score=True)\n",
    "\n",
    "    # Fit GridSearchCV\n",
    "\n",
    "    grid.fit(X_train_rfe, y_train_sample)\n",
    "\n",
    "\n",
    "    y_pred = grid.predict(X_test_rfe)\n",
    "\n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test_sample, y_pred)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "    f1 = f1_score(y_test_sample,y_pred)\n",
    "    \n",
    "    ls.append([f1,grid.best_params_,roc_auc])\n",
    "    i+=1\n",
    "\n",
    "    # Print results\n",
    "    #print(\"The ROC of the tuned classifier is\",roc_auc)\n",
    "    #print(\"The F1 of the tuned classifier is\",f1)\n",
    "\n",
    "    #print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0.56,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'sqrt',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 720}],\n",
       " [0.45652173913043476,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'sqrt',\n",
       "   'min_samples_leaf': 40,\n",
       "   'n_estimators': 898}],\n",
       " [0.5833333333333334,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'auto',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 1226}],\n",
       " [0.5915492957746479,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'sqrt',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 1458}],\n",
       " [0.6,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'sqrt',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 1035}],\n",
       " [0.5833333333333334,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'auto',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 474}],\n",
       " [0.5915492957746479,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'log2',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 1281}],\n",
       " [0.5454545454545454,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'sqrt',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 433}],\n",
       " [0.5384615384615384,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'auto',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 297}],\n",
       " [0.5915492957746479,\n",
       "  {'bootstrap': True,\n",
       "   'max_features': 'log2',\n",
       "   'min_samples_leaf': 20,\n",
       "   'n_estimators': 351}]]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(ls, columns=['F1_score', 'Parameters','ROC_AUC'])\n",
    "#ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is what I got after the time below: not impressive at all, however we are seeing something close between the score on the test and valid dataset\n",
    "\n",
    "```\n",
    "[Parallel(n_jobs=-1)]: Done 165856 tasks      | elapsed: 347.3min\n",
    "[Parallel(n_jobs=-1)]: Done 171081 tasks      | elapsed: 362.5min\n",
    "[Parallel(n_jobs=-1)]: Done 176386 tasks      | elapsed: 381.0min\n",
    "[Parallel(n_jobs=-1)]: Done 180000 out of 180000 | elapsed: 395.6min finished\n",
    "\n",
    "The ROC of the tuned classifier is 0.9757603544329325\n",
    "The F1 of the tuned classifier is 0.7407407407407407\n",
    "{'min_samples_leaf': 2, 'n_estimators': 17, 'max_depth': 9, 'max_features': 0.4, 'bootstrap': True}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have looked at this [Article](https://www.analyticsvidhya.com/blog/2015/06/tuning-random-forest-model/) to see the best parameters to tune for Random "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['roc_auc-->', 0.9981802541026998, 'F1_score-->', 0.4329896907216495, 'roc_auc_score-->', 0.9981802541026998]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.4817518248175182]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.9884804167084043]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.0071\n",
      "Accuracy  :  0.9929\n",
      "Precision  :  0.3173076923076923\n",
      "Recall  :  1.0\n",
      "Specificity  :  1.0\n",
      "False positive rate  :  0.007123507574997492\n",
      "True positive rate  :  1.0\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Test_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGeRJREFUeJzt3X2cnWV54PHflUnD8rIfE94CeSmEJVbRlqzyphWkIJCwlGClFFRIMXYsgot1V4UqpRa2i2uRhRaxg0RCsQkBrElpAoSA0F0LJC0BScLLCGJmIAQIWF6qcTL3/jFPsseUZJLhzJwr8/y+fp7PnHOf+zxzH8xnrs91Pde5nyilIElSq41o9QIkSQIDkiQpCQOSJCkFA5IkKQUDkiQpBQOSJCkFA5IkKQUDkiTVUETMioi1EfFow9ifRkR3RCyvjhMbXrswIjoj4vGIOKFhfGo11hkRFzSMT4qIB6rxmyJiVH9rMiBJUj1dD0x9k/ErSilTqmMhQEQcBJwOvKt6zzcioi0i2oCrgWnAQcAZ1VyAr1bnOhB4GZjZ34JGvsUP1K9fvPiUW0FoyOw87shWL0E107O+O5p1rmb+vfyVPQ/Y6rpKKfdFxP7beLrpwNxSys+BpyOiEziseq2zlPIUQETMBaZHxCrgGOCj1ZzZwJ8C12ztl5ghSZIanRcRj1QlvTHV2HhgdcOcrmpsS+N7AK+UUno2G98qA5IkZdG7oWlHRLRHxLKGo30bVnAN8J+AKcBzwOWD+nk3M+glO0nS0CuldAAd2/me5zc+johrgduqp93AxIapE6oxtjD+EjA6IkZWWVLj/C0yQ5KkLEpv844BiIh9G55+GNjYgbcAOD0idoqIScBk4EFgKTC56qgbRV/jw4LSdxuJe4BTq/fPAOb39/vNkCQpi96BBZKBiIg5wNHAnhHRBVwMHB0RU4AC/Bj4FEApZUVEzANWAj3AuaWUDdV5zgPuANqAWaWUFdWv+CIwNyIuBR4Crut3TYN9PyS77DSU7LLTUGtql91zq5rXZbfvO5u2rqFihiRJSZQBltqGCwOSJGUxhCW7jGxqkCSlYIYkSVlYspMkpdC7odUraClLdpKkFMyQJCkLS3aSpBTsspMkqfXMkCQpCb8YK0nKwZKdJEmtZ4YkSVlYspMkpeAXYyVJaj0zJEnKwpKdJCkFu+wkSWo9MyRJysKSnSQpBUt2kiS1nhmSJCVRSr2/h2RAkqQsan4NyZKdJCkFMyRJyqLmTQ0GJEnKouYlOwOSJGXh5qqSJLWeGZIkZWHJTpKUQs2bGizZSZJSMEOSpCws2UmSUrBkJ0lS65khSVIWNc+QDEiSlETdd/u2ZCdJSsEMSZKysGQnSUqh5m3fluwkSSmYIUlSFpbsJEkpWLKTJKn1zJAkKQtLdpKkFCzZSZLUemZIkpSFJTtJUgo1D0iW7CRJKZghSVIWNW9qMCBJUhaW7CRJaj0zJEnKwpKdJCkFS3aSJLWeGZIkZWHJTpKUgiU7SZJazwxJkrKoeYZkQJKkLEpp9QpaypKdJCkFMyRJysKSnSQphZoHJEt2kqQUzJAkKQu/GCtJSsGSnSRJrWdAkqQsSmne0Y+ImBURayPi0Yaxr0XEYxHxSET8XUSMbnjtwojojIjHI+KEhvGp1VhnRFzQMD4pIh6oxm+KiFH9rcmAJElZ9PY27+jf9cDUzcYWA+8upfwG8ARwIUBEHAScDryres83IqItItqAq4FpwEHAGdVcgK8CV5RSDgReBmb2tyADkiTVUCnlPmDdZmN3llJ6qqf3AxOqx9OBuaWUn5dSngY6gcOqo7OU8lQpZT0wF5geEQEcA9xSvX82cEp/a7KpQZKyyNXU8AngpurxePoC1EZd1RjA6s3GDwf2AF5pCG6N87fIgCRJWTSx7Tsi2oH2hqGOUkrHNr73S0AP8J2mLWgbGJAkaRiqgs82BaBGEfH7wEnAsaVs6o7oBiY2TJtQjbGF8ZeA0RExssqSGudvkdeQJCmJ0luadgxEREwFvgCcXEp5o+GlBcDpEbFTREwCJgMPAkuByVVH3Sj6Gh8WVIHsHuDU6v0zgPn9/X4zJEnKYgivIUXEHOBoYM+I6AIupq+rbidgcV9fAveXUv6wlLIiIuYBK+kr5Z1bStlQnec84A6gDZhVSllR/YovAnMj4lLgIeC6ftdUBvn+G7948al63+BDQ2rncUe2egmqmZ713dGsc73xzfOb9vdylz+8smnrGipmSJKUhXvZSZJSGOC1n+HCpgZJUgpmSJKURa4vxg45A5IkZWFAkiSlMMhdz9l5DUmSlIIZkiRlYclOg+nLf/517vu/D7L7mNF878ZvAnD1dTdy64LbGTP6bQCc/6kZHPX+wwC49oab+O5td9A2YgQX/tE5/Obh7wXg+I/MYNdddmHEiBG0tbUxb9ZVAPy3i/4nP/5JFwCvvvYa/3G33bh19tVD/TG1g+l84n5efe01NmzopaenhyPedyJ/ctHnmPmJj/LCi313JLjoostYdPvdLV5pzdS87duANMhOOfE4PvqRk/njS/7il8bP/L1TOPujp/7S2I+efoZFS+5l/o3fZO2L6/jk+RfyD3O/RVtbGwCz/vKyTUFso8svuXDT46/95bXstusug/RJNNx86Ljf5aWXXv6lsSuvupavX/HXLVqR6q7fgBQR76Dv5kwb72XRTd/measGc2HDxSFTfp3u557fprl3/+P9TDv2g4waNYoJ4/bhVyeM44ernmDKu9/Z73tLKdx+933Muuqyt7pkSa1S850attrUEBFfpO8OgEHfzq4PVo/nNN47Xdtvzq1/z4fPOocv//nX+em/vgrA2hdeYp+xe22aM3bvPVn7wosARATtf/QlTvvEZ7h5/sJ/d75/fvhR9hgzhv0m9nsPLIlSCosWzuGB+xfxyZkf2zT+6XPO5l/+eTHXdlzO6M2ycQ2B3tK8YwfUX5fdTODQUsplpZQbq+My+m5bu8X7o0dEe0Qsi4hl37phTjPXOyz83of/C4vmzeLW669mrz1252t/dW2/77nhmr/g5m//Fddcfglzvnsby5b/8JdeX7j4+5x43AcHa8kaZj74Wx/msMOnctJvf5xzzvl9jvzA4Xzzr2/g7e94P+895HjWrFnL1/7Xn7R6maqZ/gJSLzDuTcb3rV57U6WUjlLKIaWUQz551hlvZX3D0p67j6GtrY0RI0Zw6snTeHTlEwDsvdcerHn+hU3znl/7InvvtScAY6ufe4wZzbFHvZ8frnx807yeng3cde8PmHrsUUP4KbQje/bZNQC88MJLzJ+/iEMPncLatS/S29tLKYVvXfcdDj10SotXWT+lt7dpx46ov4D0WWBJRCyKiI7quB1YApw/+MsbnjZ2MQEsufcHHHjAfgD81geOYNGSe1m/fj1dz67hJ13P8uvvfDtv/NvPeP31vntlvfFvP+MHD/4Lkw/Yf9M57l/2EAfsN4F99t4LqT+77LIzu+2266bHx33og6xY8Tj77LP3pjmnTJ/GihWPb+kUGiw1L9lttamhlHJ7RLydvhJdY1PD0o03Z9LWff7iy1j60CO88sq/cuwpH+fTM89k6UOP8PiTT0HA+H3GcvEX/isABx6wHycccyQnf+xTjGxr40uf+zRtbW28tG4t5//xJQBs6NnAiccfzQeOOGTT71h0171M+9DRrfh42gGNHbsXt9zcd6+0kSPbmDv3e9xx5/e5/ttXcfDBB1FK4Zlnujjn019s8UpVN96gT8OKN+jTUGvmDfpev/TjTft7ueuXb/QGfZKkAdpBS23N4l52kqQUzJAkKYsdtDuuWQxIkpSFJTtJklrPDEmSsqj5XnYGJEnKwpKdJEmtZ4YkSUnsqHvQNYsBSZKysGQnSVLrmSFJUhY1z5AMSJKURc3bvi3ZSZJSMEOSpCws2UmSMig1D0iW7CRJKZghSVIWNc+QDEiSlEXNd2qwZCdJSsEMSZKysGQnSUqh5gHJkp0kKQUzJElKopR6Z0gGJEnKwpKdJEmtZ4YkSVnUPEMyIElSEu5lJ0lSAmZIkpRFzTMkA5IkZVHvrews2UmScjBDkqQk6t7UYECSpCxqHpAs2UmSUjBDkqQsat7UYECSpCTqfg3Jkp0kKQUzJEnKwpKdJCkDS3aSJCVghiRJWViykyRlUAxIkqQUah6QvIYkSUrBDEmSkrBkJ0nKoeYByZKdJCkFMyRJSsKSnSQphboHJEt2klRTEXF+RDwaESsi4rPV2O4RsTginqx+jqnGIyKuiojOiHgkIt7TcJ4Z1fwnI2LGQNdjQJKkJEpv847+RMS7gT8ADgMOBk6KiAOBC4AlpZTJwJLqOcA0YHJ1tAPXVOfZHbgYOLw618Ubg9j2MiBJUhYlmnf0753AA6WUN0opPcC9wO8A04HZ1ZzZwCnV4+nADaXP/cDoiNgXOAFYXEpZV0p5GVgMTB3IxzcgSdIwFBHtEbGs4WjfbMqjwJERsUdE7AKcCEwExpZSnqvmrAHGVo/HA6sb3t9VjW1pfLvZ1CBJSTSzqaGU0gF0bOX1VRHxVeBO4HVgObBhszklIobsnhhmSJKUROmNph3b9PtKua6U8t5SylHAy8ATwPNVKY7q59pqejd9GdRGE6qxLY1vNwOSJNVUROxd/fxV+q4f/S2wANjYKTcDmF89XgCcVXXbHQH8tCrt3QEcHxFjqmaG46ux7WbJTpKSaMH3kG6NiD2AXwDnllJeiYjLgHkRMRN4BjitmruQvutMncAbwNkApZR1EXEJsLSa92ellHUDWYwBSZKSKNvWHdfE31eOfJOxl4Bj32S8AOdu4TyzgFlvdT2W7CRJKZghSVISdd86yIAkSUlsa3fccGXJTpKUghmSJCVRhuwrqDkZkCQpCUt2kiQlYIYkSUnUPUMyIElSEnW/hmTJTpKUghmSJCVhyU6SlMJQ72WXjSU7SVIKZkiSlIR72UmSUui1ZCdJUuuZIUlSEnVvajAgSVISdW/7tmQnSUrBDEmSkqj71kEGJElKwpKdJEkJmCFJUhJ1/x6SAUmSkqh727clO0lSCmZIkpSEXXaSpBTqfg3Jkp0kKQUzJElKou5NDQYkSUqi7teQLNlJklIY9Axp53FHDvavkKRhoe5NDZbsJCmJul9DsmQnSUrBDEmSkrBkJ0lKoeZNdgYkScqi7hmS15AkSSmYIUlSEnXvsjMgSVISva1eQItZspMkpWCGJElJFCzZSZIS6K1537clO0lSCmZIkpREryU7SVIGdb+GZMlOkpSCGZIkJVH37yEZkCQpCUt2kiQlYIYkSUlYspMkpVD3gGTJTpKUghmSJCVR96YGA5IkJdFb73hkyU6SlIMZkiQl4V52kqQUan73CUt2kqQczJAkKYm6fw/JgCRJSfRGva8hWbKTJKVghiRJSdS9qcGAJElJ1P0akiU7SVIKZkiSlETdtw4yIElSEnXfqcGSnSTVVESMjohbIuKxiFgVEe+LiN0jYnFEPFn9HFPNjYi4KiI6I+KRiHhPw3lmVPOfjIgZA12PAUmSkihNPLbRlcDtpZR3AAcDq4ALgCWllMnAkuo5wDRgcnW0A9cARMTuwMXA4cBhwMUbg9j2MiBJUhK90byjPxHxNuAo4DqAUsr6UsorwHRgdjVtNnBK9Xg6cEPpcz8wOiL2BU4AFpdS1pVSXgYWA1MH8vkNSJI0DEVEe0QsazjaN5syCXgB+HZEPBQR34qIXYGxpZTnqjlrgLHV4/HA6ob3d1VjWxrfbjY1SFISzfweUimlA+jYypSRwHuAz5RSHoiIK/n/5bmN5ygRMWTf1zVDkqQkhvgaUhfQVUp5oHp+C30B6vmqFEf1c231ejcwseH9E6qxLY1vNwOSJNVQKWUNsDoifq0aOhZYCSwANnbKzQDmV48XAGdV3XZHAD+tSnt3AMdHxJiqmeH4amy7WbKTpCRa8MXYzwDfiYhRwFPA2fQlKvMiYibwDHBaNXchcCLQCbxRzaWUsi4iLgGWVvP+rJSybiCLMSBJUhJDvZddKWU5cMibvHTsm8wtwLlbOM8sYNZbXY8lO0lSCmZIkpRE3Xf7NiBJUhKl3lvZWbKTJOVghiRJSViykySlUPeAZMlOkpSCGZIkJTFkm8YlZUCSpCTqfgtzS3aSpBTMkCQpibo3NRiQJCmJugckS3aSpBTMkCQpCbvsJEkp1L3LzoAkSUl4DUmSpATMkCQpCa8hSZJS6K15SLJkJ0lKwQxJkpKoe1ODAUmSkqh3wc6SnSQpCTMkSUrCkp0kKYW679RgyU6SlIIZkiQlUffvIRmQJCmJeocjS3aSpCTMkCQpCbvsJEkp1P0akiU7SVIKZkiSlES98yMDkiSlUfdrSJbsJEkpmCFJUhJ1b2owIElSEvUOR5bsJElJmCFJUhJ1b2owIElSEqXmRTtLdpKkFMyQJCkJS3aSpBTq3vZtyU6SlIIZkiQlUe/8yIAkSWlYslM6Jxx/NCsevY/HVv4fvvD5c1u9HA0zEyaM4647b+aRh+/h4eV385nzZgLwkY+cxMPL72b9z1bz3vf8RotXqToyQ0pmxIgRXHXl/2DqiWfQ1fUc9//TQv7+tjtZterJVi9Nw0RPTw+f/8JXeGj5o+y22648+MDt3LXkPlaseIzfPe0PuObqy1q9xNqyy06pHHbof+ZHP/oxTz/9EwDmzZvPyb99ggFJTbNmzVrWrFkLwGuvvc5jjz3J+HH7cNeSf2zxyuQXYwcoIs5u5kLUZ9z4fVjd9eym513dzzFu3D4tXJGGs/32m8CUg9/NAw8+1OqlSG/pGtJXmrYKSUNu1113Yd5N1/K5/34xr776WquXI/pKds06dkRbLdlFxCNbegkYu5X3tQPtANH2NkaM2HXAC6ybZ7vXMHHCuE3PJ4zfl2efXdPCFWk4GjlyJDffdC1z5vwd3/veolYvR5W6l+z6u4Y0FjgBeHmz8QB+sKU3lVI6gA6AkaPG1/u/8HZaumw5Bx44if33n0h39xpOO206Z55lp52a69qOy1n1WCf/+8qOVi9F2qS/gHQbsFspZfnmL0TE9wdlRTW3YcMGzv/sl1n4D39L24gRXD/7JlaufKLVy9Iw8pvvP5QzP34qj/xwJcuW3gnARRddxqidRnHlFZey1167s2D+DTz88ApOPOljLV5tveyopbZmiVIGN4ExQ5I0nPWs745mnevM/X6naX8v/+aZ7zZtXUPFL8ZKklLwe0iSlETdy0kGJElKwr3sJElKwAxJkpLwe0iSpBTq3vZtyU6SlIIZkiQlUfemBgOSJCVR92tIluwkSSmYIUlSEjY1SJJSKKU07ehPRPyHiHgwIh6OiBUR8ZVqfFJEPBARnRFxU0SMqsZ3qp53Vq/v33CuC6vxxyPihIF+fgOSJNXTz4FjSikHA1OAqRFxBPBV4IpSyoH03XpoZjV/JvByNX5FNY+IOAg4HXgXMBX4RkS0DWRBBiRJSqKX0rSjP6XPxlsF/0p1FOAY4JZqfDZwSvV4evWc6vVjIyKq8bmllJ+XUp4GOoHDBvL5DUiSlMRQ38I8ItoiYjmwFlgM/Ah4pZTSU03pAsZXj8cDqwGq138K7NE4/ibv2S4GJElKojTxfxHRHhHLGo72f/f7StlQSpkCTKAvq3nHkH/oBnbZSdIwVErpALbpHvWllFci4h7gfcDoiBhZZUETgO5qWjcwEeiKiJHA24CXGsY3anzPdjFDkqQkhvIaUkTsFRGjq8c7A8cBq4B7gFOraTOA+dXjBdVzqtfvLn3tfAuA06suvEnAZODBgXx+MyRJSmJb2rWbaF9gdtURNwKYV0q5LSJWAnMj4lLgIeC6av51wN9ERCewjr7OOkopKyJiHrAS6AHOLaVsGMiCYrD/A4wcNb7ee2FIGtZ61ndHs841beK0pv29XLR6UdPWNVTMkCQpibrv1GBAkqQk3FxVkqQEzJAkKQnvhyRJSmGIu+zSsWQnSUrBDEmSkrBkJ0lKwS47SZISMEOSpCR6a97UYECSpCTqHY4s2UmSkjBDkqQk7LKTJKVQ94BkyU6SlIIZkiQlUfetgwxIkpSEJTtJkhIwQ5KkJOq+dZABSZKSqPs1JEt2kqQUzJAkKYm6NzUYkCQpCUt2kiQlYIYkSUlYspMkpVD3tm9LdpKkFMyQJCkJ7xgrSUrBkp0kSQmYIUlSEpbsJEkpWLKTJCkBMyRJSsKSnSQpBUt2kiQlYIYkSUlYspMkpWDJTpKkBMyQJCmJUnpbvYSWMiBJUhJ1vx+SJTtJUgpmSJKURLHLTpKUgSU7SZISMEOSpCQs2UmSUqj7Tg2W7CRJKZghSVISdd86yIAkSUl4DUmSlIJt35IkJWCGJElJWLKTJKVg27ckSQmYIUlSEpbsJEkp2GUnSVICZkiSlIQlO0lSCnbZSZKUgBmSJCXh5qqSpBQs2UmSlIAZkiQlYZedJCmFul9DsmQnSUrBgCRJSZRSmnZsi4iYGhGPR0RnRFwwyB+vX5bsJCmJobyGFBFtwNXAcUAXsDQiFpRSVg7ZIjZjhiRJ9XQY0FlKeaqUsh6YC0xv5YIMSJKURGnisQ3GA6sbnndVYy0z6CW7nvXdMdi/YziKiPZSSker16H68N9c6zXz72VEtAPtDUMd2f//NUPKq73/KVJT+W9uGCmldJRSDmk4Ng9G3cDEhucTqrGWMSBJUj0tBSZHxKSIGAWcDixo5YLsspOkGiql9ETEecAdQBswq5SyopVrMiDllbrWq2HJf3M1U0pZCCxs9To2irrvnSRJysFrSJKkFAxICWXbzkPDV0TMioi1EfFoq9ciGZCSadjOYxpwEHBGRBzU2lVpGLsemNrqRUhgQMoo3XYeGr5KKfcB61q9DgkMSBml285DkoaCAUmSlIIBKZ9023lI0lAwIOWTbjsPSRoKBqRkSik9wMbtPFYB81q9nYeGr4iYA/wT8GsR0RURM1u9JtWXOzVIklIwQ5IkpWBAkiSlYECSJKVgQJIkpWBAkiSlYECSJKVgQJIkpWBAkiSl8P8ARYs3KCR3PlUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml_ = RandomForestClassifier(n_jobs=-1, n_estimators=400, min_samples_leaf=50, max_features='auto', random_state=45)\n",
    "ml_.fit(X_train_rfe, y_train_sample)\n",
    "pred_d = ml_.predict(X_test_rfe)\n",
    "print_score_(pred_d,ml_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}