Water_Pump_Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "import category_encoders as ce\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display 100 columns for pandas\n",
    "pd.set_option('display.max_columns', 100)\n",
    "\n",
    "# Open, Read, and Save to data frame\n",
    "X_train = 'train_features.csv'\n",
    "X_test = 'test_features.csv'\n",
    "y_train = 'train_labels.csv'\n",
    "\n",
    "X_train = pd.read_csv(X_train)\n",
    "X_test = pd.read_csv(X_test)\n",
    "y_train = pd.read_csv(y_train)\n",
    "\n",
    "# Check columns to check the what type of data\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop ID for y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting status group so we drop id\n",
    "y_train = y_train.drop(columns='id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split to Training and Validation Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, random_state=42, test_size=.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encode Categorical Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode categorical features\n",
    "encoder = ce.OrdinalEncoder()\n",
    "\n",
    "# Fit & Transform\n",
    "X_train = encoder.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline Random Forest Classifier Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RandomForestClassifier(n_jobs=-1, random_state=42)\n",
    "\n",
    "# Fit training data to model\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Encode X_val categorical values\n",
    "X_val = encoder.transform(X_val)\n",
    "\n",
    "# Predict X_val\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "# Print Accuracy Score\n",
    "print('Validation Set Accuracy Score:', accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Permutation Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check which feature is unimportant by shuffling feature values\n",
    "## Weight close to 0 means it is less important and we should drop\n",
    "import eli5\n",
    "from eli5.sklearn import PermutationImportance\n",
    "\n",
    "# Instantiate. prefit means cv already done\n",
    "permuter = PermutationImportance(model, scoring='accuracy', cv='prefit', n_iter=3,\n",
    "                                 random_state=42)\n",
    "# Fit to test\n",
    "permuter.fit(X_val, y_val)\n",
    "\n",
    "# Display \n",
    "feature_names = X_val.columns.tolist()\n",
    "eli5.show_weights(permuter, top=None, feature_names=feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save features with number greater than 0\n",
    "mask = permuter.feature_importances_ > 0\n",
    "feature = X_train.columns[mask]\n",
    "X_train = X_train[feature]\n",
    "X_val = X_val[feature]\n",
    "\n",
    "# Fit training data to model\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Predict X_val\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "# Print Accuracy Score\n",
    "print('Retest Validation Set Accuracy Score:', accuracy_score(y_val, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Baseline random forest classifier gave us a pretty good score of 0.7965\n",
    "# Retest the model with permutation importance and removing columns less than 0 yield 0.7980\n",
    "# I belive we can do better by cleaning up the data and adding features\n",
    "# Check out my Water_Pump_Best notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# -----------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restart Kernal - Test with Dropped P.Imp <= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "import category_encoders as ce\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Display 100 columns for pandas\n",
    "pd.set_option('display.max_columns', 100)\n",
    "\n",
    "# Open, Read, and Save to data frame\n",
    "X_train = 'train_features.csv'\n",
    "X_test = 'test_features.csv'\n",
    "y_train = 'train_labels.csv'\n",
    "\n",
    "X_train = pd.read_csv(X_train)\n",
    "X_test = pd.read_csv(X_test)\n",
    "y_train = pd.read_csv(y_train)\n",
    "\n",
    "# Drop columns for baseline\n",
    "#drop_cols = ['recorded_by','id','num_private', 'wpt_name'] #// 1st drop, 0.79857\n",
    "# drop_cols = ['recorded_by','id','num_private', 'wpt_name', 'basin', 'source',\n",
    "##             'subvillage','management_group','source_class'] #// 2nd drop 0.79969\n",
    "#drop_cols = ['recorded_by','id','num_private', 'wpt_name', 'basin', 'source', \n",
    "##             'subvillage','management_group','source_class','water_quality','scheme_management',\n",
    "##             'permit', 'extraction_type_group','district_code'] # // 3rd drop 0.79816"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Feature Engineer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns water per person\n",
    "X_train['Water_per_person'] = X_train['amount_tsh']/X_train['population']\n",
    "X_test['Water_per_person'] = X_test['amount_tsh']/X_test['population']\n",
    "\n",
    "# Dividing by 0 creates inf values - replace with nan\n",
    "X_train['Water_per_person'] = X_train['Water_per_person'].replace([np.inf, -np.inf], np.nan)\n",
    "X_test['Water_per_person'] = X_test['Water_per_person'].replace([np.inf, -np.inf], np.nan)\n",
    "\n",
    "# Replace nan with 0\n",
    "X_train['Water_per_person'] = X_train['Water_per_person'].replace(np.nan, 0)\n",
    "X_test['Water_per_person'] = X_test['Water_per_person'].replace(np.nan, 0)\n",
    "\n",
    "# Change to panda date time to extract week and month\n",
    "X_train['date_recorded'] = pd.to_datetime(X_train['date_recorded'], infer_datetime_format=True)\n",
    "X_test['date_recorded'] = pd.to_datetime(X_test['date_recorded'], infer_datetime_format=True)\n",
    "X_train['week'] = X_train['date_recorded'].apply(lambda x: x.week)\n",
    "X_test['week'] = X_test['date_recorded'].apply(lambda x: x.week)\n",
    "X_train['month'] = X_train['date_recorded'].apply(lambda x: x.month)\n",
    "X_test['month'] = X_test['date_recorded'].apply(lambda x: x.month)\n",
    "\n",
    "# Change back to string or else RandomForestClassifier can't process\n",
    "X_train['date_recorded']= X_train['date_recorded'].astype(str)\n",
    "X_test['date_recorded'] = X_test['date_recorded'].astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns for n_estimators=400, max_depth=30 # Baseline 0.8131 // with features 0.81254\n",
    "\n",
    "drop_cols = ['recorded_by','id','num_private', 'wpt_name'] # 1st drop, 0.8122 # with features 0.8126 & test 0.81557\n",
    "#drop_cols = ['recorded_by','id','num_private', 'wpt_name','installer','date_recorded',\n",
    "#             'source','management_group','extraction_type','source_type','extraction_type_group',\n",
    "#             'district_code','lga','management','source_class','funder',\n",
    "#             'basin','scheme_management'] # 2nd drop 0.8066\n",
    "#drop_cols = ['recorded_by','id','num_private', 'wpt_name','installer','date_recorded',\n",
    "#             'source','management_group','extraction_type','source_type','extraction_type_group',\n",
    "#             'district_code','lga','management','source_class','funder',\n",
    "#             'basin','scheme_management','payment','payment_type'] # 3rd drop 0.8058\n",
    "#drop_cols = ['recorded_by','id','num_private', 'wpt_name','installer','date_recorded',\n",
    "#             'source','management_group','extraction_type','source_type','extraction_type_group',\n",
    "#             'district_code','lga','management','source_class','funder',\n",
    "#           'basin','scheme_management','payment','payment_type','quality_group'] # 4th drop 0.8054\n",
    "# drop_cols = ['recorded_by','id','num_private', 'wpt_name','installer','date_recorded',\n",
    "#              'source','management_group','extraction_type','source_type','extraction_type_group',\n",
    "#              'district_code','lga','management','source_class','funder',\n",
    "#              'basin','scheme_management','payment','payment_type',\n",
    "#              'quality_group','subvillage'] # 5th drop 0.8060"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop, Encode, Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.drop(columns=drop_cols)\n",
    "X_test = X_test.drop(columns=drop_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.drop(columns='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, random_state=42, test_size=.2)\n",
    "\n",
    "# Encode categorical features\n",
    "encoder = ce.OrdinalEncoder()\n",
    "\n",
    "# Fit & Transform\n",
    "X_train = encoder.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dustin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set Accuracy Score: 0.8126262626262626\n"
     ]
    }
   ],
   "source": [
    "model = RandomForestClassifier(n_jobs=-1, random_state=42, n_estimators=400, max_depth=30)\n",
    "\n",
    "# Fit training data to model\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Encode X_val categorical values\n",
    "X_val = encoder.transform(X_val)\n",
    "\n",
    "# Predict X_val\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "# Print Accuracy Score\n",
    "print('Validation Set Accuracy Score:', accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Permutation Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "    table.eli5-weights tr:hover {\n",
       "        filter: brightness(85%);\n",
       "    }\n",
       "</style>\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
       "    <thead>\n",
       "    <tr style=\"border: none;\">\n",
       "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
       "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
       "    </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0318\n",
       "                \n",
       "                    &plusmn; 0.0030\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                quantity\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 83.98%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0232\n",
       "                \n",
       "                    &plusmn; 0.0020\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                quantity_group\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.76%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0122\n",
       "                \n",
       "                    &plusmn; 0.0022\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                extraction_type_class\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 90.82%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0105\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                construction_year\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 91.39%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0095\n",
       "                \n",
       "                    &plusmn; 0.0003\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                waterpoint_type\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.12%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0042\n",
       "                \n",
       "                    &plusmn; 0.0019\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                longitude\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.78%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0035\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                population\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 96.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0022\n",
       "                \n",
       "                    &plusmn; 0.0004\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                latitude\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 96.96%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0022\n",
       "                \n",
       "                    &plusmn; 0.0016\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                waterpoint_type_group\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.21%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0019\n",
       "                \n",
       "                    &plusmn; 0.0017\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                amount_tsh\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.41%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0017\n",
       "                \n",
       "                    &plusmn; 0.0025\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                subvillage\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.50%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0016\n",
       "                \n",
       "                    &plusmn; 0.0008\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                scheme_name\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 98.62%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0007\n",
       "                \n",
       "                    &plusmn; 0.0004\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                region\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 98.77%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0006\n",
       "                \n",
       "                    &plusmn; 0.0025\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                gps_height\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 98.77%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0006\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                scheme_management\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.03%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0004\n",
       "                \n",
       "                    &plusmn; 0.0005\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                quality_group\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.27%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0007\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                extraction_type_group\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.43%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0002\n",
       "                \n",
       "                    &plusmn; 0.0015\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                payment\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.62%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0012\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                payment_type\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.62%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                source\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0007\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                public_meeting\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0006\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                water_quality\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0004\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                source_class\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.76%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0001\n",
       "                \n",
       "                    &plusmn; 0.0005\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                extraction_type\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.85%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0015\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                date_recorded\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.49%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0002\n",
       "                \n",
       "                    &plusmn; 0.0015\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                region_code\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.49%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0002\n",
       "                \n",
       "                    &plusmn; 0.0005\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                source_type\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.27%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0003\n",
       "                \n",
       "                    &plusmn; 0.0008\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                management_group\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.12%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0004\n",
       "                \n",
       "                    &plusmn; 0.0008\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                installer\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.12%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0004\n",
       "                \n",
       "                    &plusmn; 0.0011\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                Water_per_person\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.08%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0004\n",
       "                \n",
       "                    &plusmn; 0.0015\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                month\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.03%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0004\n",
       "                \n",
       "                    &plusmn; 0.0004\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                funder\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 99.03%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0004\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                permit\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 98.46%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0008\n",
       "                \n",
       "                    &plusmn; 0.0026\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                week\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 98.46%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0008\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                management\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 97.91%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0013\n",
       "                \n",
       "                    &plusmn; 0.0008\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                district_code\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 97.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0013\n",
       "                \n",
       "                    &plusmn; 0.0003\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                basin\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 97.60%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0015\n",
       "                \n",
       "                    &plusmn; 0.0017\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                ward\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 96.53%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                -0.0026\n",
       "                \n",
       "                    &plusmn; 0.0008\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                lga\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check which feature is unimportant by shuffling feature values\n",
    "## Weight close to 0 means it is less important and we should drop\n",
    "import eli5\n",
    "from eli5.sklearn import PermutationImportance\n",
    "\n",
    "# Instantiate. prefit means cv already done\n",
    "permuter = PermutationImportance(model, scoring='accuracy', cv='prefit', n_iter=3,\n",
    "                                 random_state=42)\n",
    "# Fit to test\n",
    "permuter.fit(X_val, y_val)\n",
    "\n",
    "# Display \n",
    "feature_names = X_val.columns.tolist()\n",
    "eli5.show_weights(permuter, top=None, feature_names=feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict and Submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode X_test\n",
    "X_test = encoder.transform(X_test)\n",
    "\n",
    "# Predict X_test\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "### SUBMISSION ###\n",
    "\n",
    "sample_submission = pd.read_csv('sample_submission.csv')\n",
    "submission = sample_submission.copy()\n",
    "submission['status_group'] = y_pred\n",
    "submission.to_csv('Baseline_2_F-submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}