Deep Learning Santander dataset with accuracy of 77.9 on kaggle.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Link to kaggle competition https://www.kaggle.com/c/santander-customer-transaction-prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.pipeline import Pipeline\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "import random\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from keras import optimizers\n",
    "from keras.constraints import maxnorm\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = pd.read_csv('train.csv')\n",
    "testing_data = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking null values in data\n",
    "training_data.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_data.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200000 entries, 0 to 199999\n",
      "Columns: 202 entries, ID_code to var_199\n",
      "dtypes: float64(200), int64(1), object(1)\n",
      "memory usage: 308.2+ MB\n"
     ]
    }
   ],
   "source": [
    "training_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>train_150932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ID_code\n",
       "count         200000\n",
       "unique        200000\n",
       "top     train_150932\n",
       "freq               1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking which is the object variable\n",
    "training_data.describe(include = ['O'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>var_5</th>\n",
       "      <th>var_6</th>\n",
       "      <th>var_7</th>\n",
       "      <th>var_8</th>\n",
       "      <th>...</th>\n",
       "      <th>var_190</th>\n",
       "      <th>var_191</th>\n",
       "      <th>var_192</th>\n",
       "      <th>var_193</th>\n",
       "      <th>var_194</th>\n",
       "      <th>var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.100490</td>\n",
       "      <td>10.679914</td>\n",
       "      <td>-1.627622</td>\n",
       "      <td>10.715192</td>\n",
       "      <td>6.796529</td>\n",
       "      <td>11.078333</td>\n",
       "      <td>-5.065317</td>\n",
       "      <td>5.408949</td>\n",
       "      <td>16.545850</td>\n",
       "      <td>0.284162</td>\n",
       "      <td>...</td>\n",
       "      <td>3.234440</td>\n",
       "      <td>7.438408</td>\n",
       "      <td>1.927839</td>\n",
       "      <td>3.331774</td>\n",
       "      <td>17.993784</td>\n",
       "      <td>-0.142088</td>\n",
       "      <td>2.303335</td>\n",
       "      <td>8.908158</td>\n",
       "      <td>15.870720</td>\n",
       "      <td>-3.326537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.300653</td>\n",
       "      <td>3.040051</td>\n",
       "      <td>4.050044</td>\n",
       "      <td>2.640894</td>\n",
       "      <td>2.043319</td>\n",
       "      <td>1.623150</td>\n",
       "      <td>7.863267</td>\n",
       "      <td>0.866607</td>\n",
       "      <td>3.418076</td>\n",
       "      <td>3.332634</td>\n",
       "      <td>...</td>\n",
       "      <td>4.559922</td>\n",
       "      <td>3.023272</td>\n",
       "      <td>1.478423</td>\n",
       "      <td>3.992030</td>\n",
       "      <td>3.135162</td>\n",
       "      <td>1.429372</td>\n",
       "      <td>5.454369</td>\n",
       "      <td>0.921625</td>\n",
       "      <td>3.010945</td>\n",
       "      <td>10.438015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.408400</td>\n",
       "      <td>-15.043400</td>\n",
       "      <td>2.117100</td>\n",
       "      <td>-0.040200</td>\n",
       "      <td>5.074800</td>\n",
       "      <td>-32.562600</td>\n",
       "      <td>2.347300</td>\n",
       "      <td>5.349700</td>\n",
       "      <td>-10.505500</td>\n",
       "      <td>...</td>\n",
       "      <td>-14.093300</td>\n",
       "      <td>-2.691700</td>\n",
       "      <td>-3.814500</td>\n",
       "      <td>-11.783400</td>\n",
       "      <td>8.694400</td>\n",
       "      <td>-5.261000</td>\n",
       "      <td>-14.209600</td>\n",
       "      <td>5.960600</td>\n",
       "      <td>6.299300</td>\n",
       "      <td>-38.852800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.453850</td>\n",
       "      <td>-4.740025</td>\n",
       "      <td>8.722475</td>\n",
       "      <td>5.254075</td>\n",
       "      <td>9.883175</td>\n",
       "      <td>-11.200350</td>\n",
       "      <td>4.767700</td>\n",
       "      <td>13.943800</td>\n",
       "      <td>-2.317800</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.058825</td>\n",
       "      <td>5.157400</td>\n",
       "      <td>0.889775</td>\n",
       "      <td>0.584600</td>\n",
       "      <td>15.629800</td>\n",
       "      <td>-1.170700</td>\n",
       "      <td>-1.946925</td>\n",
       "      <td>8.252800</td>\n",
       "      <td>13.829700</td>\n",
       "      <td>-11.208475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.524750</td>\n",
       "      <td>-1.608050</td>\n",
       "      <td>10.580000</td>\n",
       "      <td>6.825000</td>\n",
       "      <td>11.108250</td>\n",
       "      <td>-4.833150</td>\n",
       "      <td>5.385100</td>\n",
       "      <td>16.456800</td>\n",
       "      <td>0.393700</td>\n",
       "      <td>...</td>\n",
       "      <td>3.203600</td>\n",
       "      <td>7.347750</td>\n",
       "      <td>1.901300</td>\n",
       "      <td>3.396350</td>\n",
       "      <td>17.957950</td>\n",
       "      <td>-0.172700</td>\n",
       "      <td>2.408900</td>\n",
       "      <td>8.888200</td>\n",
       "      <td>15.934050</td>\n",
       "      <td>-2.819550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.758200</td>\n",
       "      <td>1.358625</td>\n",
       "      <td>12.516700</td>\n",
       "      <td>8.324100</td>\n",
       "      <td>12.261125</td>\n",
       "      <td>0.924800</td>\n",
       "      <td>6.003000</td>\n",
       "      <td>19.102900</td>\n",
       "      <td>2.937900</td>\n",
       "      <td>...</td>\n",
       "      <td>6.406200</td>\n",
       "      <td>9.512525</td>\n",
       "      <td>2.949500</td>\n",
       "      <td>6.205800</td>\n",
       "      <td>20.396525</td>\n",
       "      <td>0.829600</td>\n",
       "      <td>6.556725</td>\n",
       "      <td>9.593300</td>\n",
       "      <td>18.064725</td>\n",
       "      <td>4.836800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>20.315000</td>\n",
       "      <td>10.376800</td>\n",
       "      <td>19.353000</td>\n",
       "      <td>13.188300</td>\n",
       "      <td>16.671400</td>\n",
       "      <td>17.251600</td>\n",
       "      <td>8.447700</td>\n",
       "      <td>27.691800</td>\n",
       "      <td>10.151300</td>\n",
       "      <td>...</td>\n",
       "      <td>18.440900</td>\n",
       "      <td>16.716500</td>\n",
       "      <td>8.402400</td>\n",
       "      <td>18.281800</td>\n",
       "      <td>27.928800</td>\n",
       "      <td>4.272900</td>\n",
       "      <td>18.321500</td>\n",
       "      <td>12.000400</td>\n",
       "      <td>26.079100</td>\n",
       "      <td>28.500700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 201 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              target          var_0          var_1          var_2  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean        0.100490      10.679914      -1.627622      10.715192   \n",
       "std         0.300653       3.040051       4.050044       2.640894   \n",
       "min         0.000000       0.408400     -15.043400       2.117100   \n",
       "25%         0.000000       8.453850      -4.740025       8.722475   \n",
       "50%         0.000000      10.524750      -1.608050      10.580000   \n",
       "75%         0.000000      12.758200       1.358625      12.516700   \n",
       "max         1.000000      20.315000      10.376800      19.353000   \n",
       "\n",
       "               var_3          var_4          var_5          var_6  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean        6.796529      11.078333      -5.065317       5.408949   \n",
       "std         2.043319       1.623150       7.863267       0.866607   \n",
       "min        -0.040200       5.074800     -32.562600       2.347300   \n",
       "25%         5.254075       9.883175     -11.200350       4.767700   \n",
       "50%         6.825000      11.108250      -4.833150       5.385100   \n",
       "75%         8.324100      12.261125       0.924800       6.003000   \n",
       "max        13.188300      16.671400      17.251600       8.447700   \n",
       "\n",
       "               var_7          var_8  ...        var_190        var_191  \\\n",
       "count  200000.000000  200000.000000  ...  200000.000000  200000.000000   \n",
       "mean       16.545850       0.284162  ...       3.234440       7.438408   \n",
       "std         3.418076       3.332634  ...       4.559922       3.023272   \n",
       "min         5.349700     -10.505500  ...     -14.093300      -2.691700   \n",
       "25%        13.943800      -2.317800  ...      -0.058825       5.157400   \n",
       "50%        16.456800       0.393700  ...       3.203600       7.347750   \n",
       "75%        19.102900       2.937900  ...       6.406200       9.512525   \n",
       "max        27.691800      10.151300  ...      18.440900      16.716500   \n",
       "\n",
       "             var_192        var_193        var_194        var_195  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean        1.927839       3.331774      17.993784      -0.142088   \n",
       "std         1.478423       3.992030       3.135162       1.429372   \n",
       "min        -3.814500     -11.783400       8.694400      -5.261000   \n",
       "25%         0.889775       0.584600      15.629800      -1.170700   \n",
       "50%         1.901300       3.396350      17.957950      -0.172700   \n",
       "75%         2.949500       6.205800      20.396525       0.829600   \n",
       "max         8.402400      18.281800      27.928800       4.272900   \n",
       "\n",
       "             var_196        var_197        var_198        var_199  \n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000  \n",
       "mean        2.303335       8.908158      15.870720      -3.326537  \n",
       "std         5.454369       0.921625       3.010945      10.438015  \n",
       "min       -14.209600       5.960600       6.299300     -38.852800  \n",
       "25%        -1.946925       8.252800      13.829700     -11.208475  \n",
       "50%         2.408900       8.888200      15.934050      -2.819550  \n",
       "75%         6.556725       9.593300      18.064725       4.836800  \n",
       "max        18.321500      12.000400      26.079100      28.500700  \n",
       "\n",
       "[8 rows x 201 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>var_5</th>\n",
       "      <th>var_6</th>\n",
       "      <th>var_7</th>\n",
       "      <th>...</th>\n",
       "      <th>var_190</th>\n",
       "      <th>var_191</th>\n",
       "      <th>var_192</th>\n",
       "      <th>var_193</th>\n",
       "      <th>var_194</th>\n",
       "      <th>var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.9255</td>\n",
       "      <td>-6.7863</td>\n",
       "      <td>11.9081</td>\n",
       "      <td>5.0930</td>\n",
       "      <td>11.4607</td>\n",
       "      <td>-9.2834</td>\n",
       "      <td>5.1187</td>\n",
       "      <td>18.6266</td>\n",
       "      <td>...</td>\n",
       "      <td>4.4354</td>\n",
       "      <td>3.9642</td>\n",
       "      <td>3.1364</td>\n",
       "      <td>1.6910</td>\n",
       "      <td>18.5227</td>\n",
       "      <td>-2.3978</td>\n",
       "      <td>7.8784</td>\n",
       "      <td>8.5635</td>\n",
       "      <td>12.7803</td>\n",
       "      <td>-1.0914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_1</td>\n",
       "      <td>0</td>\n",
       "      <td>11.5006</td>\n",
       "      <td>-4.1473</td>\n",
       "      <td>13.8588</td>\n",
       "      <td>5.3890</td>\n",
       "      <td>12.3622</td>\n",
       "      <td>7.0433</td>\n",
       "      <td>5.6208</td>\n",
       "      <td>16.5338</td>\n",
       "      <td>...</td>\n",
       "      <td>7.6421</td>\n",
       "      <td>7.7214</td>\n",
       "      <td>2.5837</td>\n",
       "      <td>10.9516</td>\n",
       "      <td>15.4305</td>\n",
       "      <td>2.0339</td>\n",
       "      <td>8.1267</td>\n",
       "      <td>8.7889</td>\n",
       "      <td>18.3560</td>\n",
       "      <td>1.9518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_2</td>\n",
       "      <td>0</td>\n",
       "      <td>8.6093</td>\n",
       "      <td>-2.7457</td>\n",
       "      <td>12.0805</td>\n",
       "      <td>7.8928</td>\n",
       "      <td>10.5825</td>\n",
       "      <td>-9.0837</td>\n",
       "      <td>6.9427</td>\n",
       "      <td>14.6155</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9057</td>\n",
       "      <td>9.7905</td>\n",
       "      <td>1.6704</td>\n",
       "      <td>1.6858</td>\n",
       "      <td>21.6042</td>\n",
       "      <td>3.1417</td>\n",
       "      <td>-6.5213</td>\n",
       "      <td>8.2675</td>\n",
       "      <td>14.7222</td>\n",
       "      <td>0.3965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_3</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0604</td>\n",
       "      <td>-2.1518</td>\n",
       "      <td>8.9522</td>\n",
       "      <td>7.1957</td>\n",
       "      <td>12.5846</td>\n",
       "      <td>-1.8361</td>\n",
       "      <td>5.8428</td>\n",
       "      <td>14.9250</td>\n",
       "      <td>...</td>\n",
       "      <td>4.4666</td>\n",
       "      <td>4.7433</td>\n",
       "      <td>0.7178</td>\n",
       "      <td>1.4214</td>\n",
       "      <td>23.0347</td>\n",
       "      <td>-1.2706</td>\n",
       "      <td>-2.9275</td>\n",
       "      <td>10.2922</td>\n",
       "      <td>17.9697</td>\n",
       "      <td>-8.9996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_4</td>\n",
       "      <td>0</td>\n",
       "      <td>9.8369</td>\n",
       "      <td>-1.4834</td>\n",
       "      <td>12.8746</td>\n",
       "      <td>6.6375</td>\n",
       "      <td>12.2772</td>\n",
       "      <td>2.4486</td>\n",
       "      <td>5.9405</td>\n",
       "      <td>19.2514</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.4905</td>\n",
       "      <td>9.5214</td>\n",
       "      <td>-0.1508</td>\n",
       "      <td>9.1942</td>\n",
       "      <td>13.2876</td>\n",
       "      <td>-1.5121</td>\n",
       "      <td>3.9267</td>\n",
       "      <td>9.5031</td>\n",
       "      <td>17.9974</td>\n",
       "      <td>-8.8104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 202 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID_code  target    var_0   var_1    var_2   var_3    var_4   var_5   var_6  \\\n",
       "0  train_0       0   8.9255 -6.7863  11.9081  5.0930  11.4607 -9.2834  5.1187   \n",
       "1  train_1       0  11.5006 -4.1473  13.8588  5.3890  12.3622  7.0433  5.6208   \n",
       "2  train_2       0   8.6093 -2.7457  12.0805  7.8928  10.5825 -9.0837  6.9427   \n",
       "3  train_3       0  11.0604 -2.1518   8.9522  7.1957  12.5846 -1.8361  5.8428   \n",
       "4  train_4       0   9.8369 -1.4834  12.8746  6.6375  12.2772  2.4486  5.9405   \n",
       "\n",
       "     var_7  ...  var_190  var_191  var_192  var_193  var_194  var_195  \\\n",
       "0  18.6266  ...   4.4354   3.9642   3.1364   1.6910  18.5227  -2.3978   \n",
       "1  16.5338  ...   7.6421   7.7214   2.5837  10.9516  15.4305   2.0339   \n",
       "2  14.6155  ...   2.9057   9.7905   1.6704   1.6858  21.6042   3.1417   \n",
       "3  14.9250  ...   4.4666   4.7433   0.7178   1.4214  23.0347  -1.2706   \n",
       "4  19.2514  ...  -1.4905   9.5214  -0.1508   9.1942  13.2876  -1.5121   \n",
       "\n",
       "   var_196  var_197  var_198  var_199  \n",
       "0   7.8784   8.5635  12.7803  -1.0914  \n",
       "1   8.1267   8.7889  18.3560   1.9518  \n",
       "2  -6.5213   8.2675  14.7222   0.3965  \n",
       "3  -2.9275  10.2922  17.9697  -8.9996  \n",
       "4   3.9267   9.5031  17.9974  -8.8104  \n",
       "\n",
       "[5 rows x 202 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.89951\n",
       "1    0.10049\n",
       "Name: target, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This looks like a inbalance dataset\n",
    "\n",
    "training_data['target'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 202)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>var_5</th>\n",
       "      <th>var_6</th>\n",
       "      <th>var_7</th>\n",
       "      <th>var_8</th>\n",
       "      <th>...</th>\n",
       "      <th>var_190</th>\n",
       "      <th>var_191</th>\n",
       "      <th>var_192</th>\n",
       "      <th>var_193</th>\n",
       "      <th>var_194</th>\n",
       "      <th>var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>11.0656</td>\n",
       "      <td>7.7798</td>\n",
       "      <td>12.9536</td>\n",
       "      <td>9.4292</td>\n",
       "      <td>11.4327</td>\n",
       "      <td>-2.3805</td>\n",
       "      <td>5.8493</td>\n",
       "      <td>18.2675</td>\n",
       "      <td>2.1337</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.1556</td>\n",
       "      <td>11.8495</td>\n",
       "      <td>-1.4300</td>\n",
       "      <td>2.4508</td>\n",
       "      <td>13.7112</td>\n",
       "      <td>2.4669</td>\n",
       "      <td>4.3654</td>\n",
       "      <td>10.7200</td>\n",
       "      <td>15.4722</td>\n",
       "      <td>-8.7197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>8.5304</td>\n",
       "      <td>1.2543</td>\n",
       "      <td>11.3047</td>\n",
       "      <td>5.1858</td>\n",
       "      <td>9.1974</td>\n",
       "      <td>-4.0117</td>\n",
       "      <td>6.0196</td>\n",
       "      <td>18.6316</td>\n",
       "      <td>-4.4131</td>\n",
       "      <td>...</td>\n",
       "      <td>10.6165</td>\n",
       "      <td>8.8349</td>\n",
       "      <td>0.9403</td>\n",
       "      <td>10.1282</td>\n",
       "      <td>15.5765</td>\n",
       "      <td>0.4773</td>\n",
       "      <td>-1.4852</td>\n",
       "      <td>9.8714</td>\n",
       "      <td>19.1293</td>\n",
       "      <td>-20.9760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>5.4827</td>\n",
       "      <td>-10.3581</td>\n",
       "      <td>10.1407</td>\n",
       "      <td>7.0479</td>\n",
       "      <td>10.2628</td>\n",
       "      <td>9.8052</td>\n",
       "      <td>4.8950</td>\n",
       "      <td>20.2537</td>\n",
       "      <td>1.5233</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.7484</td>\n",
       "      <td>10.9935</td>\n",
       "      <td>1.9803</td>\n",
       "      <td>2.1800</td>\n",
       "      <td>12.9813</td>\n",
       "      <td>2.1281</td>\n",
       "      <td>-7.1086</td>\n",
       "      <td>7.0618</td>\n",
       "      <td>19.8956</td>\n",
       "      <td>-23.1794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>8.5374</td>\n",
       "      <td>-1.3222</td>\n",
       "      <td>12.0220</td>\n",
       "      <td>6.5749</td>\n",
       "      <td>8.8458</td>\n",
       "      <td>3.1744</td>\n",
       "      <td>4.9397</td>\n",
       "      <td>20.5660</td>\n",
       "      <td>3.3755</td>\n",
       "      <td>...</td>\n",
       "      <td>9.5702</td>\n",
       "      <td>9.0766</td>\n",
       "      <td>1.6580</td>\n",
       "      <td>3.5813</td>\n",
       "      <td>15.1874</td>\n",
       "      <td>3.1656</td>\n",
       "      <td>3.9567</td>\n",
       "      <td>9.2295</td>\n",
       "      <td>13.0168</td>\n",
       "      <td>-4.2108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>11.7058</td>\n",
       "      <td>-0.1327</td>\n",
       "      <td>14.1295</td>\n",
       "      <td>7.7506</td>\n",
       "      <td>9.1035</td>\n",
       "      <td>-8.5848</td>\n",
       "      <td>6.8595</td>\n",
       "      <td>10.6048</td>\n",
       "      <td>2.9890</td>\n",
       "      <td>...</td>\n",
       "      <td>4.2259</td>\n",
       "      <td>9.1723</td>\n",
       "      <td>1.2835</td>\n",
       "      <td>3.3778</td>\n",
       "      <td>19.5542</td>\n",
       "      <td>-0.2860</td>\n",
       "      <td>-5.1612</td>\n",
       "      <td>7.2882</td>\n",
       "      <td>13.9260</td>\n",
       "      <td>-9.1846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 201 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    var_0    var_1    var_2   var_3    var_4   var_5   var_6  \\\n",
       "0  test_0  11.0656   7.7798  12.9536  9.4292  11.4327 -2.3805  5.8493   \n",
       "1  test_1   8.5304   1.2543  11.3047  5.1858   9.1974 -4.0117  6.0196   \n",
       "2  test_2   5.4827 -10.3581  10.1407  7.0479  10.2628  9.8052  4.8950   \n",
       "3  test_3   8.5374  -1.3222  12.0220  6.5749   8.8458  3.1744  4.9397   \n",
       "4  test_4  11.7058  -0.1327  14.1295  7.7506   9.1035 -8.5848  6.8595   \n",
       "\n",
       "     var_7   var_8  ...  var_190  var_191  var_192  var_193  var_194  var_195  \\\n",
       "0  18.2675  2.1337  ...  -2.1556  11.8495  -1.4300   2.4508  13.7112   2.4669   \n",
       "1  18.6316 -4.4131  ...  10.6165   8.8349   0.9403  10.1282  15.5765   0.4773   \n",
       "2  20.2537  1.5233  ...  -0.7484  10.9935   1.9803   2.1800  12.9813   2.1281   \n",
       "3  20.5660  3.3755  ...   9.5702   9.0766   1.6580   3.5813  15.1874   3.1656   \n",
       "4  10.6048  2.9890  ...   4.2259   9.1723   1.2835   3.3778  19.5542  -0.2860   \n",
       "\n",
       "   var_196  var_197  var_198  var_199  \n",
       "0   4.3654  10.7200  15.4722  -8.7197  \n",
       "1  -1.4852   9.8714  19.1293 -20.9760  \n",
       "2  -7.1086   7.0618  19.8956 -23.1794  \n",
       "3   3.9567   9.2295  13.0168  -4.2108  \n",
       "4  -5.1612   7.2882  13.9260  -9.1846  \n",
       "\n",
       "[5 rows x 201 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200000 entries, 0 to 199999\n",
      "Columns: 201 entries, ID_code to var_199\n",
      "dtypes: float64(200), object(1)\n",
      "memory usage: 306.7+ MB\n"
     ]
    }
   ],
   "source": [
    "testing_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 201)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>var_5</th>\n",
       "      <th>var_6</th>\n",
       "      <th>var_7</th>\n",
       "      <th>...</th>\n",
       "      <th>var_190</th>\n",
       "      <th>var_191</th>\n",
       "      <th>var_192</th>\n",
       "      <th>var_193</th>\n",
       "      <th>var_194</th>\n",
       "      <th>var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>199995</th>\n",
       "      <td>train_199995</td>\n",
       "      <td>0</td>\n",
       "      <td>11.4880</td>\n",
       "      <td>-0.4956</td>\n",
       "      <td>8.2622</td>\n",
       "      <td>3.5142</td>\n",
       "      <td>10.3404</td>\n",
       "      <td>11.6081</td>\n",
       "      <td>5.6709</td>\n",
       "      <td>15.1516</td>\n",
       "      <td>...</td>\n",
       "      <td>6.1415</td>\n",
       "      <td>13.2305</td>\n",
       "      <td>3.9901</td>\n",
       "      <td>0.9388</td>\n",
       "      <td>18.0249</td>\n",
       "      <td>-1.7939</td>\n",
       "      <td>2.1661</td>\n",
       "      <td>8.5326</td>\n",
       "      <td>16.6660</td>\n",
       "      <td>-17.8661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199996</th>\n",
       "      <td>train_199996</td>\n",
       "      <td>0</td>\n",
       "      <td>4.9149</td>\n",
       "      <td>-2.4484</td>\n",
       "      <td>16.7052</td>\n",
       "      <td>6.6345</td>\n",
       "      <td>8.3096</td>\n",
       "      <td>-10.5628</td>\n",
       "      <td>5.8802</td>\n",
       "      <td>21.5940</td>\n",
       "      <td>...</td>\n",
       "      <td>4.9611</td>\n",
       "      <td>4.6549</td>\n",
       "      <td>0.6998</td>\n",
       "      <td>1.8341</td>\n",
       "      <td>22.2717</td>\n",
       "      <td>1.7337</td>\n",
       "      <td>-2.1651</td>\n",
       "      <td>6.7419</td>\n",
       "      <td>15.9054</td>\n",
       "      <td>0.3388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199997</th>\n",
       "      <td>train_199997</td>\n",
       "      <td>0</td>\n",
       "      <td>11.2232</td>\n",
       "      <td>-5.0518</td>\n",
       "      <td>10.5127</td>\n",
       "      <td>5.6456</td>\n",
       "      <td>9.3410</td>\n",
       "      <td>-5.4086</td>\n",
       "      <td>4.5555</td>\n",
       "      <td>21.5571</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0651</td>\n",
       "      <td>5.4414</td>\n",
       "      <td>3.1032</td>\n",
       "      <td>4.8793</td>\n",
       "      <td>23.5311</td>\n",
       "      <td>-1.5736</td>\n",
       "      <td>1.2832</td>\n",
       "      <td>8.7155</td>\n",
       "      <td>13.8329</td>\n",
       "      <td>4.1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199998</th>\n",
       "      <td>train_199998</td>\n",
       "      <td>0</td>\n",
       "      <td>9.7148</td>\n",
       "      <td>-8.6098</td>\n",
       "      <td>13.6104</td>\n",
       "      <td>5.7930</td>\n",
       "      <td>12.5173</td>\n",
       "      <td>0.5339</td>\n",
       "      <td>6.0479</td>\n",
       "      <td>17.0152</td>\n",
       "      <td>...</td>\n",
       "      <td>2.6840</td>\n",
       "      <td>8.6587</td>\n",
       "      <td>2.7337</td>\n",
       "      <td>11.1178</td>\n",
       "      <td>20.4158</td>\n",
       "      <td>-0.0786</td>\n",
       "      <td>6.7980</td>\n",
       "      <td>10.0342</td>\n",
       "      <td>15.5289</td>\n",
       "      <td>-13.9001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199999</th>\n",
       "      <td>train_199999</td>\n",
       "      <td>0</td>\n",
       "      <td>10.8762</td>\n",
       "      <td>-5.7105</td>\n",
       "      <td>12.1183</td>\n",
       "      <td>8.0328</td>\n",
       "      <td>11.5577</td>\n",
       "      <td>0.3488</td>\n",
       "      <td>5.2839</td>\n",
       "      <td>15.2058</td>\n",
       "      <td>...</td>\n",
       "      <td>8.9842</td>\n",
       "      <td>1.6893</td>\n",
       "      <td>0.1276</td>\n",
       "      <td>0.3766</td>\n",
       "      <td>15.2101</td>\n",
       "      <td>-2.4907</td>\n",
       "      <td>-2.2342</td>\n",
       "      <td>8.1857</td>\n",
       "      <td>12.1284</td>\n",
       "      <td>0.1385</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 202 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             ID_code  target    var_0   var_1    var_2   var_3    var_4  \\\n",
       "199995  train_199995       0  11.4880 -0.4956   8.2622  3.5142  10.3404   \n",
       "199996  train_199996       0   4.9149 -2.4484  16.7052  6.6345   8.3096   \n",
       "199997  train_199997       0  11.2232 -5.0518  10.5127  5.6456   9.3410   \n",
       "199998  train_199998       0   9.7148 -8.6098  13.6104  5.7930  12.5173   \n",
       "199999  train_199999       0  10.8762 -5.7105  12.1183  8.0328  11.5577   \n",
       "\n",
       "          var_5   var_6    var_7  ...  var_190  var_191  var_192  var_193  \\\n",
       "199995  11.6081  5.6709  15.1516  ...   6.1415  13.2305   3.9901   0.9388   \n",
       "199996 -10.5628  5.8802  21.5940  ...   4.9611   4.6549   0.6998   1.8341   \n",
       "199997  -5.4086  4.5555  21.5571  ...   4.0651   5.4414   3.1032   4.8793   \n",
       "199998   0.5339  6.0479  17.0152  ...   2.6840   8.6587   2.7337  11.1178   \n",
       "199999   0.3488  5.2839  15.2058  ...   8.9842   1.6893   0.1276   0.3766   \n",
       "\n",
       "        var_194  var_195  var_196  var_197  var_198  var_199  \n",
       "199995  18.0249  -1.7939   2.1661   8.5326  16.6660 -17.8661  \n",
       "199996  22.2717   1.7337  -2.1651   6.7419  15.9054   0.3388  \n",
       "199997  23.5311  -1.5736   1.2832   8.7155  13.8329   4.1995  \n",
       "199998  20.4158  -0.0786   6.7980  10.0342  15.5289 -13.9001  \n",
       "199999  15.2101  -2.4907  -2.2342   8.1857  12.1284   0.1385  \n",
       "\n",
       "[5 rows x 202 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_predictors = training_data.drop(['target', 'ID_code'], axis = 1)\n",
    "\n",
    "training_target = training_data['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, val_X, train_y, val_y = train_test_split(training_predictors, training_target, test_size = 0.25,\n",
    "                                                 stratify = training_target, random_state = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.3037 - acc: 0.8995 - val_loss: 0.2681 - val_acc: 0.9039\n",
      "Epoch 2/20\n",
      "150000/150000 [==============================] - 4s 27us/step - loss: 0.2674 - acc: 0.9045 - val_loss: 0.2639 - val_acc: 0.9067\n",
      "Epoch 3/20\n",
      "150000/150000 [==============================] - 4s 26us/step - loss: 0.2632 - acc: 0.9056 - val_loss: 0.2589 - val_acc: 0.9037\n",
      "Epoch 4/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2602 - acc: 0.9062 - val_loss: 0.2706 - val_acc: 0.9035\n",
      "Epoch 5/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2586 - acc: 0.9076 - val_loss: 0.3094 - val_acc: 0.9050\n",
      "Epoch 6/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2580 - acc: 0.9076 - val_loss: 0.2548 - val_acc: 0.9080\n",
      "Epoch 7/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2568 - acc: 0.9082 - val_loss: 0.2730 - val_acc: 0.9100\n",
      "Epoch 8/20\n",
      "150000/150000 [==============================] - 4s 25us/step - loss: 0.2572 - acc: 0.9088 - val_loss: 0.2663 - val_acc: 0.9088\n",
      "Epoch 9/20\n",
      "150000/150000 [==============================] - 4s 25us/step - loss: 0.2570 - acc: 0.9089 - val_loss: 0.2820 - val_acc: 0.9098\n",
      "Epoch 10/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2559 - acc: 0.9101 - val_loss: 0.2652 - val_acc: 0.9086\n",
      "Epoch 11/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2538 - acc: 0.9099 - val_loss: 0.2591 - val_acc: 0.9078\n",
      "Epoch 12/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2525 - acc: 0.9104 - val_loss: 0.2804 - val_acc: 0.9065\n",
      "Epoch 13/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2521 - acc: 0.9109 - val_loss: 0.2532 - val_acc: 0.9100\n",
      "Epoch 14/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2553 - acc: 0.9115 - val_loss: 0.2793 - val_acc: 0.9102\n",
      "Epoch 15/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2532 - acc: 0.9116 - val_loss: 0.2620 - val_acc: 0.9094\n",
      "Epoch 16/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2558 - acc: 0.9118 - val_loss: 0.2823 - val_acc: 0.9047\n",
      "Epoch 17/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2531 - acc: 0.9113 - val_loss: 0.3114 - val_acc: 0.9103\n",
      "Epoch 18/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2497 - acc: 0.9122 - val_loss: 0.2844 - val_acc: 0.9108\n",
      "Epoch 19/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2508 - acc: 0.9130 - val_loss: 0.2704 - val_acc: 0.9100\n",
      "Epoch 20/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.2514 - acc: 0.9121 - val_loss: 0.2749 - val_acc: 0.9098\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fe9399f5f8>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(150, activation='relu', input_shape=(training_predictors.shape[1],)))\n",
    "model.add(layers.Dense(100, activation='relu'))\n",
    "model.add(layers.Dense(50, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "#history = \n",
    "model.fit(train_X, \n",
    "            train_y,\n",
    "            epochs=20,\n",
    "            batch_size=128,\n",
    "            validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "std = StandardScaler()\n",
    "train_X = std.fit_transform(train_X)\n",
    "val_X = std.fit_transform(val_X)\n",
    "testing_predictor_transformed = std.transform(testing_data.drop(['ID_code'], axis = 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.2515 - acc: 0.9085 - val_loss: 0.2436 - val_acc: 0.9106\n",
      "Epoch 2/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.2327 - acc: 0.9143 - val_loss: 0.2452 - val_acc: 0.9104\n",
      "Epoch 3/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.2224 - acc: 0.9179 - val_loss: 0.2460 - val_acc: 0.9101\n",
      "Epoch 4/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.2114 - acc: 0.9217 - val_loss: 0.2589 - val_acc: 0.9091\n",
      "Epoch 5/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.1987 - acc: 0.9272 - val_loss: 0.2714 - val_acc: 0.9071\n",
      "Epoch 6/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.1853 - acc: 0.9321 - val_loss: 0.2907 - val_acc: 0.9043\n",
      "Epoch 7/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1730 - acc: 0.9372 - val_loss: 0.3066 - val_acc: 0.9016\n",
      "Epoch 8/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.1629 - acc: 0.9413 - val_loss: 0.3294 - val_acc: 0.9006\n",
      "Epoch 9/20\n",
      "150000/150000 [==============================] - 4s 24us/step - loss: 0.1530 - acc: 0.9453 - val_loss: 0.3539 - val_acc: 0.8968\n",
      "Epoch 10/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1430 - acc: 0.9483 - val_loss: 0.3698 - val_acc: 0.8891\n",
      "Epoch 11/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1352 - acc: 0.9516 - val_loss: 0.3991 - val_acc: 0.8953\n",
      "Epoch 12/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1275 - acc: 0.9544 - val_loss: 0.4680 - val_acc: 0.8782\n",
      "Epoch 13/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1209 - acc: 0.9566 - val_loss: 0.4683 - val_acc: 0.8847\n",
      "Epoch 14/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1150 - acc: 0.9590 - val_loss: 0.5193 - val_acc: 0.8845\n",
      "Epoch 15/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.1099 - acc: 0.9608 - val_loss: 0.5134 - val_acc: 0.8862\n",
      "Epoch 16/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.1041 - acc: 0.9628 - val_loss: 0.5262 - val_acc: 0.8829\n",
      "Epoch 17/20\n",
      "150000/150000 [==============================] - 3s 22us/step - loss: 0.1004 - acc: 0.9642 - val_loss: 0.5659 - val_acc: 0.8888\n",
      "Epoch 18/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.0956 - acc: 0.9658 - val_loss: 0.6204 - val_acc: 0.8873\n",
      "Epoch 19/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.0922 - acc: 0.9671 - val_loss: 0.6425 - val_acc: 0.8907\n",
      "Epoch 20/20\n",
      "150000/150000 [==============================] - 3s 23us/step - loss: 0.0890 - acc: 0.9684 - val_loss: 0.6646 - val_acc: 0.8809\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fe93469438>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(150, activation='relu', input_shape=(training_predictors.shape[1],)))\n",
    "model.add(layers.Dense(100, activation='relu'))\n",
    "model.add(layers.Dense(50, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_X, \n",
    "            train_y,\n",
    "            epochs=20,\n",
    "            batch_size=128,\n",
    "            validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/20\n",
      "150000/150000 [==============================] - 9s 59us/step - loss: 0.3330 - acc: 0.8983 - val_loss: 0.3188 - val_acc: 0.8995\n",
      "Epoch 2/20\n",
      "150000/150000 [==============================] - 8s 56us/step - loss: 0.3007 - acc: 0.8995 - val_loss: 0.2747 - val_acc: 0.8995\n",
      "Epoch 3/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2757 - acc: 0.8999 - val_loss: 0.2491 - val_acc: 0.9042\n",
      "Epoch 4/20\n",
      "150000/150000 [==============================] - 8s 54us/step - loss: 0.2626 - acc: 0.9023 - val_loss: 0.2509 - val_acc: 0.9107\n",
      "Epoch 5/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2560 - acc: 0.9036 - val_loss: 0.2521 - val_acc: 0.9110\n",
      "Epoch 6/20\n",
      "150000/150000 [==============================] - 8s 56us/step - loss: 0.2524 - acc: 0.9054 - val_loss: 0.2457 - val_acc: 0.9089\n",
      "Epoch 7/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2489 - acc: 0.9061 - val_loss: 0.2505 - val_acc: 0.9074\n",
      "Epoch 8/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2471 - acc: 0.9069 - val_loss: 0.2453 - val_acc: 0.9071\n",
      "Epoch 9/20\n",
      "150000/150000 [==============================] - 9s 58us/step - loss: 0.2458 - acc: 0.9077 - val_loss: 0.2465 - val_acc: 0.9097\n",
      "Epoch 10/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2443 - acc: 0.9082 - val_loss: 0.2474 - val_acc: 0.9061\n",
      "Epoch 11/20\n",
      "150000/150000 [==============================] - 9s 57us/step - loss: 0.2430 - acc: 0.9086 - val_loss: 0.2485 - val_acc: 0.9049\n",
      "Epoch 12/20\n",
      "150000/150000 [==============================] - 8s 54us/step - loss: 0.2417 - acc: 0.9090 - val_loss: 0.2485 - val_acc: 0.9066\n",
      "Epoch 13/20\n",
      "150000/150000 [==============================] - 8s 53us/step - loss: 0.2413 - acc: 0.9088 - val_loss: 0.2432 - val_acc: 0.9074\n",
      "Epoch 14/20\n",
      "150000/150000 [==============================] - 8s 54us/step - loss: 0.2399 - acc: 0.9096 - val_loss: 0.2450 - val_acc: 0.9047\n",
      "Epoch 15/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2390 - acc: 0.9105 - val_loss: 0.2457 - val_acc: 0.9047\n",
      "Epoch 16/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2386 - acc: 0.9102 - val_loss: 0.2484 - val_acc: 0.9046\n",
      "Epoch 17/20\n",
      "150000/150000 [==============================] - 9s 59us/step - loss: 0.2382 - acc: 0.9104 - val_loss: 0.2490 - val_acc: 0.9019\n",
      "Epoch 18/20\n",
      "150000/150000 [==============================] - 8s 54us/step - loss: 0.2377 - acc: 0.9103 - val_loss: 0.2448 - val_acc: 0.9039\n",
      "Epoch 19/20\n",
      "150000/150000 [==============================] - 8s 55us/step - loss: 0.2371 - acc: 0.9102 - val_loss: 0.2465 - val_acc: 0.9031\n",
      "Epoch 20/20\n",
      "150000/150000 [==============================] - 8s 54us/step - loss: 0.2366 - acc: 0.9108 - val_loss: 0.2453 - val_acc: 0.9046\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fe94143c50>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd = optimizers.SGD(lr = 0.05)\n",
    "model = models.Sequential()\n",
    "\n",
    "model.add(layers.Dense(250, activation='relu', input_shape=(training_predictors.shape[1],)))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(180, activation='relu'))\n",
    "model.add(layers.Dropout(0.40))\n",
    "model.add(layers.Dense(120, activation='relu'))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(70, activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(30, activation = 'relu'))\n",
    "model.add(layers.Dense(10, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(optimizer=sgd,\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_X, \n",
    "            train_y,\n",
    "            epochs=20,\n",
    "            batch_size=128,\n",
    "            validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/20\n",
      "150000/150000 [==============================] - 10s 64us/step - loss: 0.3366 - acc: 0.8990 - val_loss: 0.3263 - val_acc: 0.8995\n",
      "Epoch 2/20\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.3277 - acc: 0.8995 - val_loss: 0.3259 - val_acc: 0.8995\n",
      "Epoch 3/20\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.3264 - acc: 0.8995 - val_loss: 0.3248 - val_acc: 0.8995\n",
      "Epoch 4/20\n",
      "150000/150000 [==============================] - 10s 63us/step - loss: 0.3188 - acc: 0.8996 - val_loss: 0.2803 - val_acc: 0.8995\n",
      "Epoch 5/20\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.2718 - acc: 0.9031 - val_loss: 0.2458 - val_acc: 0.9051\n",
      "Epoch 6/20\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.2549 - acc: 0.9060 - val_loss: 0.2409 - val_acc: 0.9092\n",
      "Epoch 7/20\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.2495 - acc: 0.9067 - val_loss: 0.2411 - val_acc: 0.9093\n",
      "Epoch 8/20\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.2475 - acc: 0.9073 - val_loss: 0.2444 - val_acc: 0.9061\n",
      "Epoch 9/20\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.2458 - acc: 0.9074 - val_loss: 0.2380 - val_acc: 0.9124\n",
      "Epoch 10/20\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.2446 - acc: 0.9078 - val_loss: 0.2407 - val_acc: 0.9105\n",
      "Epoch 11/20\n",
      "150000/150000 [==============================] - 10s 67us/step - loss: 0.2427 - acc: 0.9081 - val_loss: 0.2399 - val_acc: 0.9114\n",
      "Epoch 12/20\n",
      "150000/150000 [==============================] - 10s 66us/step - loss: 0.2413 - acc: 0.9085 - val_loss: 0.2416 - val_acc: 0.9100\n",
      "Epoch 13/20\n",
      "150000/150000 [==============================] - 10s 67us/step - loss: 0.2409 - acc: 0.9086 - val_loss: 0.2427 - val_acc: 0.9091\n",
      "Epoch 14/20\n",
      "150000/150000 [==============================] - 11s 72us/step - loss: 0.2399 - acc: 0.9088 - val_loss: 0.2462 - val_acc: 0.9062\n",
      "Epoch 15/20\n",
      "150000/150000 [==============================] - 10s 66us/step - loss: 0.2397 - acc: 0.9087 - val_loss: 0.2456 - val_acc: 0.9065\n",
      "Epoch 16/20\n",
      "150000/150000 [==============================] - 10s 67us/step - loss: 0.2379 - acc: 0.9095 - val_loss: 0.2498 - val_acc: 0.9041\n",
      "Epoch 17/20\n",
      "150000/150000 [==============================] - 10s 68us/step - loss: 0.2375 - acc: 0.9098 - val_loss: 0.2447 - val_acc: 0.9100\n",
      "Epoch 18/20\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.2367 - acc: 0.9100 - val_loss: 0.2456 - val_acc: 0.9089\n",
      "Epoch 19/20\n",
      "150000/150000 [==============================] - 10s 64us/step - loss: 0.2353 - acc: 0.9104 - val_loss: 0.2470 - val_acc: 0.9085\n",
      "Epoch 20/20\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.2353 - acc: 0.9104 - val_loss: 0.2474 - val_acc: 0.9086\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fe9490a320>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd = optimizers.SGD(lr = 0.07)\n",
    "np.random.seed(7)\n",
    "model = models.Sequential()\n",
    "\n",
    "model.add(layers.Dense(250, activation='relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), input_shape=(training_predictors.shape[1],)))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(180,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation='relu'))\n",
    "model.add(layers.Dropout(0.40))\n",
    "model.add(layers.Dense(120,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation='relu'))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(70,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(30,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(optimizer=sgd,\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_X, \n",
    "            train_y,\n",
    "            epochs=20,\n",
    "            batch_size=128,\n",
    "            validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/9\n",
      "150000/150000 [==============================] - 10s 65us/step - loss: 0.3366 - acc: 0.8990 - val_loss: 0.3263 - val_acc: 0.8995\n",
      "Epoch 2/9\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.3277 - acc: 0.8995 - val_loss: 0.3259 - val_acc: 0.8995\n",
      "Epoch 3/9\n",
      "150000/150000 [==============================] - 9s 61us/step - loss: 0.3264 - acc: 0.8995 - val_loss: 0.3248 - val_acc: 0.8995\n",
      "Epoch 4/9\n",
      "150000/150000 [==============================] - 11s 71us/step - loss: 0.3188 - acc: 0.8996 - val_loss: 0.2806 - val_acc: 0.8995\n",
      "Epoch 5/9\n",
      "150000/150000 [==============================] - 10s 65us/step - loss: 0.2719 - acc: 0.9030 - val_loss: 0.2457 - val_acc: 0.9051\n",
      "Epoch 6/9\n",
      "150000/150000 [==============================] - 10s 67us/step - loss: 0.2550 - acc: 0.9059 - val_loss: 0.2408 - val_acc: 0.9091\n",
      "Epoch 7/9\n",
      "150000/150000 [==============================] - 9s 60us/step - loss: 0.2495 - acc: 0.9067 - val_loss: 0.2409 - val_acc: 0.9093\n",
      "Epoch 8/9\n",
      "150000/150000 [==============================] - 10s 64us/step - loss: 0.2474 - acc: 0.9072 - val_loss: 0.2443 - val_acc: 0.9062\n",
      "Epoch 9/9\n",
      "150000/150000 [==============================] - 9s 62us/step - loss: 0.2458 - acc: 0.9073 - val_loss: 0.2379 - val_acc: 0.9125\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fe952a9f28>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(7)\n",
    "model = models.Sequential()\n",
    "\n",
    "model.add(layers.Dense(250, activation='relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), input_shape=(training_predictors.shape[1],)))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(180,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation='relu'))\n",
    "model.add(layers.Dropout(0.40))\n",
    "model.add(layers.Dense(120,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation='relu'))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(70,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(30,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(optimizer=sgd,\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.fit(train_X, \n",
    "            train_y.values,\n",
    "            epochs=9,\n",
    "            batch_size=128,\n",
    "            validation_data=(val_X, val_y.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(testing_predictor_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# got score of 77.9\n",
    "answers = pd.DataFrame({'ID_code':testing_data['ID_code'], 'target':predictions.reshape(200000,)})\n",
    "answers.to_csv('deeppredictions.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}