HIGGS Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HIGGS Dataset\n",
    "See https://archive.ics.uci.edu/ml/datasets/HIGGS for dataset information and feature descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import pickle\n",
    "import operator\n",
    "import glob\n",
    "from scipy.io.arff import loadarff \n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "import seaborn as sns; sns.set_style('white')\n",
    "\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import accuracy_score, plot_confusion_matrix, f1_score, plot_roc_curve, roc_auc_score, make_scorer\n",
    "from sklearn.model_selection import KFold, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('HIGGS.csv', header=None)\n",
    "df.columns = ['class', 'lepton_pT', 'lepton_eta', 'lepton_phi', 'missing_energy_magnitude', \n",
    "              'missing_energy_phi', 'jet_1_pt', 'jet_1_eta', 'jet_1_phi', 'jet_1_b-tag', \n",
    "              'jet_2_pt', 'jet_2_eta', 'jet_2_phi', 'jet_2_b-tag', 'jet_3_pt', 'jet_3_eta', \n",
    "              'jet_3_phi', 'jet_3_b-tag', 'jet_4_pt', 'jet_4_eta', 'jet_4_phi', 'jet_4_b-tag', \n",
    "              'm_jj', 'm_jjj', 'm_lv', 'm_jlv', 'm_bb', 'm_wbb', 'm_wwbb']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>lepton_pT</th>\n",
       "      <th>lepton_eta</th>\n",
       "      <th>lepton_phi</th>\n",
       "      <th>missing_energy_magnitude</th>\n",
       "      <th>missing_energy_phi</th>\n",
       "      <th>jet_1_pt</th>\n",
       "      <th>jet_1_eta</th>\n",
       "      <th>jet_1_phi</th>\n",
       "      <th>jet_1_b-tag</th>\n",
       "      <th>...</th>\n",
       "      <th>jet_4_eta</th>\n",
       "      <th>jet_4_phi</th>\n",
       "      <th>jet_4_b-tag</th>\n",
       "      <th>m_jj</th>\n",
       "      <th>m_jjj</th>\n",
       "      <th>m_lv</th>\n",
       "      <th>m_jlv</th>\n",
       "      <th>m_bb</th>\n",
       "      <th>m_wbb</th>\n",
       "      <th>m_wwbb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.869293</td>\n",
       "      <td>-0.635082</td>\n",
       "      <td>0.225690</td>\n",
       "      <td>0.327470</td>\n",
       "      <td>-0.689993</td>\n",
       "      <td>0.754202</td>\n",
       "      <td>-0.248573</td>\n",
       "      <td>-1.092064</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.010455</td>\n",
       "      <td>-0.045767</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>1.353760</td>\n",
       "      <td>0.979563</td>\n",
       "      <td>0.978076</td>\n",
       "      <td>0.920005</td>\n",
       "      <td>0.721657</td>\n",
       "      <td>0.988751</td>\n",
       "      <td>0.876678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.907542</td>\n",
       "      <td>0.329147</td>\n",
       "      <td>0.359412</td>\n",
       "      <td>1.497970</td>\n",
       "      <td>-0.313010</td>\n",
       "      <td>1.095531</td>\n",
       "      <td>-0.557525</td>\n",
       "      <td>-1.588230</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.138930</td>\n",
       "      <td>-0.000819</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.302220</td>\n",
       "      <td>0.833048</td>\n",
       "      <td>0.985700</td>\n",
       "      <td>0.978098</td>\n",
       "      <td>0.779732</td>\n",
       "      <td>0.992356</td>\n",
       "      <td>0.798343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.798835</td>\n",
       "      <td>1.470639</td>\n",
       "      <td>-1.635975</td>\n",
       "      <td>0.453773</td>\n",
       "      <td>0.425629</td>\n",
       "      <td>1.104875</td>\n",
       "      <td>1.282322</td>\n",
       "      <td>1.381664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.128848</td>\n",
       "      <td>0.900461</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.909753</td>\n",
       "      <td>1.108330</td>\n",
       "      <td>0.985692</td>\n",
       "      <td>0.951331</td>\n",
       "      <td>0.803252</td>\n",
       "      <td>0.865924</td>\n",
       "      <td>0.780118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.344385</td>\n",
       "      <td>-0.876626</td>\n",
       "      <td>0.935913</td>\n",
       "      <td>1.992050</td>\n",
       "      <td>0.882454</td>\n",
       "      <td>1.786066</td>\n",
       "      <td>-1.646778</td>\n",
       "      <td>-0.942383</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.678379</td>\n",
       "      <td>-1.360356</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.946652</td>\n",
       "      <td>1.028704</td>\n",
       "      <td>0.998656</td>\n",
       "      <td>0.728281</td>\n",
       "      <td>0.869200</td>\n",
       "      <td>1.026736</td>\n",
       "      <td>0.957904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.105009</td>\n",
       "      <td>0.321356</td>\n",
       "      <td>1.522401</td>\n",
       "      <td>0.882808</td>\n",
       "      <td>-1.205349</td>\n",
       "      <td>0.681466</td>\n",
       "      <td>-1.070464</td>\n",
       "      <td>-0.921871</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.373566</td>\n",
       "      <td>0.113041</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.755856</td>\n",
       "      <td>1.361057</td>\n",
       "      <td>0.986610</td>\n",
       "      <td>0.838085</td>\n",
       "      <td>1.133295</td>\n",
       "      <td>0.872245</td>\n",
       "      <td>0.808487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.159912</td>\n",
       "      <td>1.013847</td>\n",
       "      <td>0.108615</td>\n",
       "      <td>1.495524</td>\n",
       "      <td>-0.537545</td>\n",
       "      <td>2.342396</td>\n",
       "      <td>-0.839740</td>\n",
       "      <td>1.320683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.097068</td>\n",
       "      <td>1.190680</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>0.822136</td>\n",
       "      <td>0.766772</td>\n",
       "      <td>1.002191</td>\n",
       "      <td>1.061233</td>\n",
       "      <td>0.837004</td>\n",
       "      <td>0.860472</td>\n",
       "      <td>0.772484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999996</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.618388</td>\n",
       "      <td>-1.012982</td>\n",
       "      <td>1.110139</td>\n",
       "      <td>0.941023</td>\n",
       "      <td>-0.379199</td>\n",
       "      <td>1.004656</td>\n",
       "      <td>0.348535</td>\n",
       "      <td>-1.678593</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.216995</td>\n",
       "      <td>1.049177</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>0.826829</td>\n",
       "      <td>0.989809</td>\n",
       "      <td>1.029104</td>\n",
       "      <td>1.199679</td>\n",
       "      <td>0.891481</td>\n",
       "      <td>0.938490</td>\n",
       "      <td>0.865269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999997</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.700559</td>\n",
       "      <td>0.774251</td>\n",
       "      <td>1.520182</td>\n",
       "      <td>0.847112</td>\n",
       "      <td>0.211230</td>\n",
       "      <td>1.095531</td>\n",
       "      <td>0.052457</td>\n",
       "      <td>0.024553</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>1.585235</td>\n",
       "      <td>1.713962</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.337374</td>\n",
       "      <td>0.845208</td>\n",
       "      <td>0.987610</td>\n",
       "      <td>0.883422</td>\n",
       "      <td>1.888438</td>\n",
       "      <td>1.153766</td>\n",
       "      <td>0.931279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.178030</td>\n",
       "      <td>0.117796</td>\n",
       "      <td>-1.276980</td>\n",
       "      <td>1.864457</td>\n",
       "      <td>-0.584370</td>\n",
       "      <td>0.998519</td>\n",
       "      <td>-1.264549</td>\n",
       "      <td>1.276333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.399515</td>\n",
       "      <td>-1.313189</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.838842</td>\n",
       "      <td>0.882890</td>\n",
       "      <td>1.201380</td>\n",
       "      <td>0.939216</td>\n",
       "      <td>0.339705</td>\n",
       "      <td>0.759070</td>\n",
       "      <td>0.719119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.464477</td>\n",
       "      <td>-0.337047</td>\n",
       "      <td>0.229019</td>\n",
       "      <td>0.954596</td>\n",
       "      <td>-0.868466</td>\n",
       "      <td>0.430004</td>\n",
       "      <td>-0.271348</td>\n",
       "      <td>-1.252278</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.652782</td>\n",
       "      <td>-0.586254</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.752535</td>\n",
       "      <td>0.740727</td>\n",
       "      <td>0.986917</td>\n",
       "      <td>0.663952</td>\n",
       "      <td>0.576084</td>\n",
       "      <td>0.541427</td>\n",
       "      <td>0.517420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11000000 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          class  lepton_pT  lepton_eta  lepton_phi  missing_energy_magnitude  \\\n",
       "0           1.0   0.869293   -0.635082    0.225690                  0.327470   \n",
       "1           1.0   0.907542    0.329147    0.359412                  1.497970   \n",
       "2           1.0   0.798835    1.470639   -1.635975                  0.453773   \n",
       "3           0.0   1.344385   -0.876626    0.935913                  1.992050   \n",
       "4           1.0   1.105009    0.321356    1.522401                  0.882808   \n",
       "...         ...        ...         ...         ...                       ...   \n",
       "10999995    1.0   1.159912    1.013847    0.108615                  1.495524   \n",
       "10999996    1.0   0.618388   -1.012982    1.110139                  0.941023   \n",
       "10999997    1.0   0.700559    0.774251    1.520182                  0.847112   \n",
       "10999998    0.0   1.178030    0.117796   -1.276980                  1.864457   \n",
       "10999999    0.0   0.464477   -0.337047    0.229019                  0.954596   \n",
       "\n",
       "          missing_energy_phi  jet_1_pt  jet_1_eta  jet_1_phi  jet_1_b-tag  \\\n",
       "0                  -0.689993  0.754202  -0.248573  -1.092064     0.000000   \n",
       "1                  -0.313010  1.095531  -0.557525  -1.588230     2.173076   \n",
       "2                   0.425629  1.104875   1.282322   1.381664     0.000000   \n",
       "3                   0.882454  1.786066  -1.646778  -0.942383     0.000000   \n",
       "4                  -1.205349  0.681466  -1.070464  -0.921871     0.000000   \n",
       "...                      ...       ...        ...        ...          ...   \n",
       "10999995           -0.537545  2.342396  -0.839740   1.320683     0.000000   \n",
       "10999996           -0.379199  1.004656   0.348535  -1.678593     2.173076   \n",
       "10999997            0.211230  1.095531   0.052457   0.024553     2.173076   \n",
       "10999998           -0.584370  0.998519  -1.264549   1.276333     0.000000   \n",
       "10999999           -0.868466  0.430004  -0.271348  -1.252278     2.173076   \n",
       "\n",
       "          ...  jet_4_eta  jet_4_phi  jet_4_b-tag      m_jj     m_jjj  \\\n",
       "0         ...  -0.010455  -0.045767     3.101961  1.353760  0.979563   \n",
       "1         ...  -1.138930  -0.000819     0.000000  0.302220  0.833048   \n",
       "2         ...   1.128848   0.900461     0.000000  0.909753  1.108330   \n",
       "3         ...  -0.678379  -1.360356     0.000000  0.946652  1.028704   \n",
       "4         ...  -0.373566   0.113041     0.000000  0.755856  1.361057   \n",
       "...       ...        ...        ...          ...       ...       ...   \n",
       "10999995  ...  -0.097068   1.190680     3.101961  0.822136  0.766772   \n",
       "10999996  ...  -0.216995   1.049177     3.101961  0.826829  0.989809   \n",
       "10999997  ...   1.585235   1.713962     0.000000  0.337374  0.845208   \n",
       "10999998  ...   1.399515  -1.313189     0.000000  0.838842  0.882890   \n",
       "10999999  ...  -1.652782  -0.586254     0.000000  0.752535  0.740727   \n",
       "\n",
       "              m_lv     m_jlv      m_bb     m_wbb    m_wwbb  \n",
       "0         0.978076  0.920005  0.721657  0.988751  0.876678  \n",
       "1         0.985700  0.978098  0.779732  0.992356  0.798343  \n",
       "2         0.985692  0.951331  0.803252  0.865924  0.780118  \n",
       "3         0.998656  0.728281  0.869200  1.026736  0.957904  \n",
       "4         0.986610  0.838085  1.133295  0.872245  0.808487  \n",
       "...            ...       ...       ...       ...       ...  \n",
       "10999995  1.002191  1.061233  0.837004  0.860472  0.772484  \n",
       "10999996  1.029104  1.199679  0.891481  0.938490  0.865269  \n",
       "10999997  0.987610  0.883422  1.888438  1.153766  0.931279  \n",
       "10999998  1.201380  0.939216  0.339705  0.759070  0.719119  \n",
       "10999999  0.986917  0.663952  0.576084  0.541427  0.517420  \n",
       "\n",
       "[11000000 rows x 29 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    6000\n",
       "0.0    6000\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separate majority and minority classes\n",
    "df_majority = df[df['class']==0]\n",
    "df_minority = df[df['class']==1]\n",
    "\n",
    "# Downsample majority and minority class\n",
    "df_majority_downsampled = resample(df_majority, \n",
    "                                 replace=False,    # sample without replacement\n",
    "                                 n_samples=6000,     # to match minority class\n",
    "                                 random_state=123) # reproducible results\n",
    "\n",
    "df_minority_downsampled = resample(df_minority, \n",
    "                                 replace=False,    # sample with replacement\n",
    "                                 n_samples=6000,     # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine minority class with downsampled majority class\n",
    "df_downsampled = pd.concat([df_majority_downsampled, df_minority_downsampled])\n",
    " \n",
    "# Display new class counts\n",
    "df_downsampled['class'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>lepton_pT</th>\n",
       "      <th>lepton_eta</th>\n",
       "      <th>lepton_phi</th>\n",
       "      <th>missing_energy_magnitude</th>\n",
       "      <th>missing_energy_phi</th>\n",
       "      <th>jet_1_pt</th>\n",
       "      <th>jet_1_eta</th>\n",
       "      <th>jet_1_phi</th>\n",
       "      <th>jet_1_b-tag</th>\n",
       "      <th>...</th>\n",
       "      <th>jet_4_eta</th>\n",
       "      <th>jet_4_phi</th>\n",
       "      <th>jet_4_b-tag</th>\n",
       "      <th>m_jj</th>\n",
       "      <th>m_jjj</th>\n",
       "      <th>m_lv</th>\n",
       "      <th>m_jlv</th>\n",
       "      <th>m_bb</th>\n",
       "      <th>m_wbb</th>\n",
       "      <th>m_wwbb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2984264</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.558361</td>\n",
       "      <td>0.125588</td>\n",
       "      <td>-0.383650</td>\n",
       "      <td>0.499787</td>\n",
       "      <td>-1.464681</td>\n",
       "      <td>0.656732</td>\n",
       "      <td>-0.140638</td>\n",
       "      <td>0.583364</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.719931</td>\n",
       "      <td>0.224578</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>1.373681</td>\n",
       "      <td>0.927890</td>\n",
       "      <td>0.994193</td>\n",
       "      <td>1.126345</td>\n",
       "      <td>1.159692</td>\n",
       "      <td>0.833733</td>\n",
       "      <td>0.796250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10764116</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.296253</td>\n",
       "      <td>0.516149</td>\n",
       "      <td>-0.117317</td>\n",
       "      <td>0.636398</td>\n",
       "      <td>-0.455267</td>\n",
       "      <td>0.988075</td>\n",
       "      <td>0.144548</td>\n",
       "      <td>0.907119</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>1.291248</td>\n",
       "      <td>1.573014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.901899</td>\n",
       "      <td>0.875995</td>\n",
       "      <td>0.975326</td>\n",
       "      <td>0.882177</td>\n",
       "      <td>1.255846</td>\n",
       "      <td>0.909731</td>\n",
       "      <td>0.763828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9530004</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.062368</td>\n",
       "      <td>-0.338021</td>\n",
       "      <td>1.577887</td>\n",
       "      <td>0.534542</td>\n",
       "      <td>-0.183894</td>\n",
       "      <td>1.081973</td>\n",
       "      <td>-1.038777</td>\n",
       "      <td>-0.608094</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>0.164438</td>\n",
       "      <td>0.940969</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.120079</td>\n",
       "      <td>0.906208</td>\n",
       "      <td>1.043838</td>\n",
       "      <td>0.997955</td>\n",
       "      <td>1.427019</td>\n",
       "      <td>0.901307</td>\n",
       "      <td>0.789998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5455217</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.355182</td>\n",
       "      <td>0.181104</td>\n",
       "      <td>0.414898</td>\n",
       "      <td>0.758879</td>\n",
       "      <td>1.102309</td>\n",
       "      <td>0.513184</td>\n",
       "      <td>-0.193120</td>\n",
       "      <td>1.042387</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>0.516723</td>\n",
       "      <td>1.498656</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>0.959572</td>\n",
       "      <td>0.989931</td>\n",
       "      <td>0.984073</td>\n",
       "      <td>0.778193</td>\n",
       "      <td>0.565135</td>\n",
       "      <td>0.838409</td>\n",
       "      <td>0.813519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6082563</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.659565</td>\n",
       "      <td>-1.691838</td>\n",
       "      <td>0.469274</td>\n",
       "      <td>0.860520</td>\n",
       "      <td>-0.639793</td>\n",
       "      <td>0.850298</td>\n",
       "      <td>-1.143741</td>\n",
       "      <td>1.576804</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.006511</td>\n",
       "      <td>-1.012426</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.901754</td>\n",
       "      <td>1.060937</td>\n",
       "      <td>0.985931</td>\n",
       "      <td>0.842516</td>\n",
       "      <td>0.874739</td>\n",
       "      <td>0.762080</td>\n",
       "      <td>0.752349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>521000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.165585</td>\n",
       "      <td>1.037223</td>\n",
       "      <td>-1.130496</td>\n",
       "      <td>1.259846</td>\n",
       "      <td>-0.187387</td>\n",
       "      <td>0.397484</td>\n",
       "      <td>-0.461473</td>\n",
       "      <td>-0.793256</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.586680</td>\n",
       "      <td>1.607974</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>1.355938</td>\n",
       "      <td>1.000538</td>\n",
       "      <td>1.261941</td>\n",
       "      <td>1.162553</td>\n",
       "      <td>0.474319</td>\n",
       "      <td>0.771688</td>\n",
       "      <td>1.002877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1672790</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.706049</td>\n",
       "      <td>0.738214</td>\n",
       "      <td>1.374808</td>\n",
       "      <td>0.757621</td>\n",
       "      <td>-0.172691</td>\n",
       "      <td>0.392995</td>\n",
       "      <td>0.581240</td>\n",
       "      <td>0.354407</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.128715</td>\n",
       "      <td>-0.656173</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.957407</td>\n",
       "      <td>1.225450</td>\n",
       "      <td>0.997393</td>\n",
       "      <td>0.571817</td>\n",
       "      <td>1.134243</td>\n",
       "      <td>0.831666</td>\n",
       "      <td>0.737647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7951892</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.741004</td>\n",
       "      <td>-0.879548</td>\n",
       "      <td>0.831599</td>\n",
       "      <td>0.560017</td>\n",
       "      <td>-0.590536</td>\n",
       "      <td>1.122463</td>\n",
       "      <td>-0.517916</td>\n",
       "      <td>-1.707975</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.693370</td>\n",
       "      <td>0.101387</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>0.807574</td>\n",
       "      <td>1.053625</td>\n",
       "      <td>0.986507</td>\n",
       "      <td>0.659439</td>\n",
       "      <td>0.759948</td>\n",
       "      <td>0.887652</td>\n",
       "      <td>0.748781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10444720</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.245926</td>\n",
       "      <td>-0.610733</td>\n",
       "      <td>-0.588952</td>\n",
       "      <td>0.719146</td>\n",
       "      <td>0.448291</td>\n",
       "      <td>1.028657</td>\n",
       "      <td>0.864445</td>\n",
       "      <td>1.571815</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.700865</td>\n",
       "      <td>1.082472</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>0.727640</td>\n",
       "      <td>0.912113</td>\n",
       "      <td>1.055416</td>\n",
       "      <td>0.650828</td>\n",
       "      <td>0.919102</td>\n",
       "      <td>0.956839</td>\n",
       "      <td>0.801805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5729388</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.383404</td>\n",
       "      <td>-1.080185</td>\n",
       "      <td>-0.242160</td>\n",
       "      <td>1.824291</td>\n",
       "      <td>1.061444</td>\n",
       "      <td>2.387559</td>\n",
       "      <td>1.132798</td>\n",
       "      <td>-0.476151</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.177019</td>\n",
       "      <td>-0.231108</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.058214</td>\n",
       "      <td>0.788499</td>\n",
       "      <td>1.069115</td>\n",
       "      <td>1.305123</td>\n",
       "      <td>0.820215</td>\n",
       "      <td>1.079734</td>\n",
       "      <td>1.016867</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12000 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          class  lepton_pT  lepton_eta  lepton_phi  missing_energy_magnitude  \\\n",
       "2984264     0.0   0.558361    0.125588   -0.383650                  0.499787   \n",
       "10764116    0.0   1.296253    0.516149   -0.117317                  0.636398   \n",
       "9530004     0.0   1.062368   -0.338021    1.577887                  0.534542   \n",
       "5455217     0.0   1.355182    0.181104    0.414898                  0.758879   \n",
       "6082563     0.0   0.659565   -1.691838    0.469274                  0.860520   \n",
       "...         ...        ...         ...         ...                       ...   \n",
       "521000      1.0   1.165585    1.037223   -1.130496                  1.259846   \n",
       "1672790     1.0   0.706049    0.738214    1.374808                  0.757621   \n",
       "7951892     1.0   0.741004   -0.879548    0.831599                  0.560017   \n",
       "10444720    1.0   1.245926   -0.610733   -0.588952                  0.719146   \n",
       "5729388     1.0   0.383404   -1.080185   -0.242160                  1.824291   \n",
       "\n",
       "          missing_energy_phi  jet_1_pt  jet_1_eta  jet_1_phi  jet_1_b-tag  \\\n",
       "2984264            -1.464681  0.656732  -0.140638   0.583364     0.000000   \n",
       "10764116           -0.455267  0.988075   0.144548   0.907119     2.173076   \n",
       "9530004            -0.183894  1.081973  -1.038777  -0.608094     2.173076   \n",
       "5455217             1.102309  0.513184  -0.193120   1.042387     2.173076   \n",
       "6082563            -0.639793  0.850298  -1.143741   1.576804     2.173076   \n",
       "...                      ...       ...        ...        ...          ...   \n",
       "521000             -0.187387  0.397484  -0.461473  -0.793256     0.000000   \n",
       "1672790            -0.172691  0.392995   0.581240   0.354407     2.173076   \n",
       "7951892            -0.590536  1.122463  -0.517916  -1.707975     0.000000   \n",
       "10444720            0.448291  1.028657   0.864445   1.571815     2.173076   \n",
       "5729388             1.061444  2.387559   1.132798  -0.476151     0.000000   \n",
       "\n",
       "          ...  jet_4_eta  jet_4_phi  jet_4_b-tag      m_jj     m_jjj  \\\n",
       "2984264   ...   0.719931   0.224578     3.101961  1.373681  0.927890   \n",
       "10764116  ...   1.291248   1.573014     0.000000  0.901899  0.875995   \n",
       "9530004   ...   0.164438   0.940969     0.000000  1.120079  0.906208   \n",
       "5455217   ...   0.516723   1.498656     3.101961  0.959572  0.989931   \n",
       "6082563   ...  -1.006511  -1.012426     0.000000  0.901754  1.060937   \n",
       "...       ...        ...        ...          ...       ...       ...   \n",
       "521000    ...   0.586680   1.607974     3.101961  1.355938  1.000538   \n",
       "1672790   ...  -0.128715  -0.656173     0.000000  0.957407  1.225450   \n",
       "7951892   ...  -0.693370   0.101387     3.101961  0.807574  1.053625   \n",
       "10444720  ...  -0.700865   1.082472     3.101961  0.727640  0.912113   \n",
       "5729388   ...  -0.177019  -0.231108     0.000000  1.058214  0.788499   \n",
       "\n",
       "              m_lv     m_jlv      m_bb     m_wbb    m_wwbb  \n",
       "2984264   0.994193  1.126345  1.159692  0.833733  0.796250  \n",
       "10764116  0.975326  0.882177  1.255846  0.909731  0.763828  \n",
       "9530004   1.043838  0.997955  1.427019  0.901307  0.789998  \n",
       "5455217   0.984073  0.778193  0.565135  0.838409  0.813519  \n",
       "6082563   0.985931  0.842516  0.874739  0.762080  0.752349  \n",
       "...            ...       ...       ...       ...       ...  \n",
       "521000    1.261941  1.162553  0.474319  0.771688  1.002877  \n",
       "1672790   0.997393  0.571817  1.134243  0.831666  0.737647  \n",
       "7951892   0.986507  0.659439  0.759948  0.887652  0.748781  \n",
       "10444720  1.055416  0.650828  0.919102  0.956839  0.801805  \n",
       "5729388   1.069115  1.305123  0.820215  1.079734  1.016867  \n",
       "\n",
       "[12000 rows x 29 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_downsampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_downsampled.copy()\n",
    "scaler = StandardScaler()\n",
    "X, y = df.iloc[:,1:].to_numpy(), df.iloc[:,0].to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Search & Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment():\n",
    "    pipeline1 = Pipeline((\n",
    "    ('clf', RandomForestClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline2 = Pipeline((\n",
    "    ('clf', KNeighborsClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline3 = Pipeline((\n",
    "    ('clf', AdaBoostClassifier()),\n",
    "    ))\n",
    "    \n",
    "    pipeline4 = Pipeline((\n",
    "    ('clf', LogisticRegression()),\n",
    "    ))\n",
    "    \n",
    "    pipeline5 = Pipeline((\n",
    "    ('clf', MLPClassifier()),\n",
    "    ))\n",
    "    \n",
    "    # Random Forest\n",
    "    parameters1 = {\n",
    "    'clf__n_estimators': [1024],\n",
    "    'clf__max_features': [1, 2, 4, 6, 8, 12, 16, 20]\n",
    "    }\n",
    "\n",
    "    # KNN\n",
    "    parameters2 = {\n",
    "    'clf__n_neighbors': [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,65,69,73,77,81,85,89,93,97,101,105],\n",
    "    'clf__weights': ['uniform', 'distance']\n",
    "    }\n",
    "    \n",
    "    # AdaBoost (Boosted Decision Tree)\n",
    "    parameters3 = {\n",
    "        'clf__algorithm': ['SAMME.R'],\n",
    "        'clf__n_estimators': [2,4,8,16,32,64,128,256,512,1024,2048],\n",
    "        'clf__learning_rate': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 2e1, 5e1]\n",
    "    }\n",
    "    \n",
    "    # Logistic\n",
    "    parameters4 = {\n",
    "    'clf__penalty':['l1', 'l2', None],\n",
    "    'clf__C':[1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0, 1e0, 1e1, 1e2, 1e3, 1e4],\n",
    "    'clf__max_iter':[5000]\n",
    "    }\n",
    "    \n",
    "    # Multi-layer Perceptron\n",
    "    parameters5 = {\n",
    "        'clf__hidden_layer_sizes':[(1,), (2,), (4,), (8,), (32,), (128,)],\n",
    "        'clf__solver':['sgd'],\n",
    "        'clf__activation':['relu'], \n",
    "        'clf__learning_rate':['constant', 'invscaling'], \n",
    "        'clf__learning_rate_init': [1e-3, 1e-2, 1e-1, 1e0],\n",
    "        'clf__max_iter': [2, 4, 8, 16, 32, 64, 128, 256, 512]\n",
    "    }\n",
    "    \n",
    "    pars = [parameters1, parameters2, parameters3, parameters4, parameters5]\n",
    "    pips = [pipeline1, pipeline2, pipeline3, pipeline4, pipeline5]\n",
    "    \n",
    "    # List of dictionaries to hold the scores of the various metrics for each type of classifier\n",
    "    best_clf_list = []\n",
    "    trial_storage = {}\n",
    "    training_storage = {}\n",
    "    \n",
    "    print(\"starting Gridsearch\")\n",
    "    for i in range(len(pars)):\n",
    "        trial_averages = []\n",
    "        train_performance = []\n",
    "        for t in range(5):\n",
    "            # split and scale data\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/6, random_state=t)\n",
    "            X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.5, random_state=t)\n",
    "            X_train = scaler.fit_transform(X_train)\n",
    "            X_val = scaler.transform(X_val)\n",
    "            X_test = scaler.transform(X_test)\n",
    "                \n",
    "            clf = GridSearchCV(pips[i], pars[i], refit=False, n_jobs=8, cv=5, verbose=3, scoring=('accuracy', 'roc_auc', 'f1'))\n",
    "            clf = clf.fit(X_val, y_val)\n",
    "            \n",
    "            print(\"finished Gridsearch trial \" + str(t + 1) + \" classifier \" + str(i + 1))\n",
    "            print(\"\")\n",
    "            print(\"\")\n",
    "            \n",
    "            # find the best params for each metric in a given trial \n",
    "            best_index_acc = np.argmin(clf.cv_results_['rank_test_accuracy'])\n",
    "            best_params_acc = clf.cv_results_['params'][best_index_acc]\n",
    "            best_index_roc = np.argmin(clf.cv_results_['rank_test_roc_auc'])\n",
    "            best_params_roc = clf.cv_results_['params'][best_index_roc]\n",
    "            best_index_f1 = np.argmin(clf.cv_results_['rank_test_f1'])\n",
    "            best_params_f1 = clf.cv_results_['params'][best_index_f1]\n",
    "    \n",
    "            # train and test models for given metric with their corresponding best parameter settings\n",
    "            pipe = pips[i]\n",
    "            clf_acc = pipe.set_params(**best_params_acc)\n",
    "            clf_acc = clf_acc.fit(X_train, y_train)\n",
    "            clf_roc = pipe.set_params(**best_params_roc)\n",
    "            clf_roc = clf_roc.fit(X_train, y_train)\n",
    "            clf_f1 = pipe.set_params(**best_params_f1)\n",
    "            clf_f1 = clf_f1.fit(X_train, y_train)\n",
    "            \n",
    "            # get training set performance\n",
    "            train_acc = accuracy_score(y_train, clf_acc.predict(X_train))\n",
    "            train_roc = roc_auc_score(y_train, clf_roc.predict_proba(X_train)[:, 1])\n",
    "            train_f1 = f1_score(y_train, clf_f1.predict(X_train))\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1,\n",
    "                'average': (train_f1 + train_acc + train_roc)/3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1 score': train_f1\n",
    "            })\n",
    "            \n",
    "            # get test set performances \n",
    "            trial_acc = clf_acc.score(X_test, y_test)\n",
    "            trial_roc = roc_auc_score(y_test, clf_roc.predict_proba(X_test)[:, 1])\n",
    "            trial_f1 = f1_score(y_test, clf_f1.predict(X_test))\n",
    "            \n",
    "            # store scores and their averages in list containing averages for each trial\n",
    "            trial_averages.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(trial_acc + trial_roc + trial_f1) / 3,\n",
    "                'accuracy': trial_acc,\n",
    "                'roc_auc_score': trial_roc,\n",
    "                'f1_score': trial_f1\n",
    "            })\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(train_acc + train_roc + train_f1) / 3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1_score': train_f1\n",
    "            })\n",
    "            \n",
    "        # find the trial with the best average metric scores and append those scores as a dict to best clf list\n",
    "        max_average = 0\n",
    "        for trial in trial_averages:\n",
    "            if trial['average'] > max_average:\n",
    "                max_average = trial['average']\n",
    "                best_trial = trial\n",
    "\n",
    "        best_clf_list.append(best_trial)\n",
    "        training_storage[str(i + 1)]=train_performance\n",
    "        trial_storage[str(i + 1)]=trial_averages\n",
    "    \n",
    "    return best_clf_list, trial_storage, training_storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting Gridsearch\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 1 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 2 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 3 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 4 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 5 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 1 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 2 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 3 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 4 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 5 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 1 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 2 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 3 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 4 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 5 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 1 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 2 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 3 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 4 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 5 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 1 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 2 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 3 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 4 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 5 classifier 5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%capture --no-stdout --no-display\n",
    "best_clf_list, trial_storage, training_perf = experiment()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating and Organizing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Models On Average For Test Set:\n",
      "{'Model #': 1, 'average': 0.7561565748411576, 'accuracy': 0.735, 'roc_auc_score': 0.8038778877887788, 'f1_score': 0.7295918367346939}\n",
      "{'Model #': 2, 'average': 0.6618301953002318, 'accuracy': 0.627, 'roc_auc_score': 0.6856835683568356, 'f1_score': 0.6728070175438597}\n",
      "{'Model #': 3, 'average': 0.7288081064743643, 'accuracy': 0.705, 'roc_auc_score': 0.7714931493149316, 'f1_score': 0.7099311701081613}\n",
      "{'Model #': 4, 'average': 0.6415731580835212, 'accuracy': 0.62, 'roc_auc_score': 0.668008193179818, 'f1_score': 0.6367112810707457}\n",
      "{'Model #': 5, 'average': 0.6889371606935969, 'accuracy': 0.671, 'roc_auc_score': 0.7281347144040231, 'f1_score': 0.6676767676767676}\n",
      "\n",
      "Train Set Data\n",
      "[{'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 1, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}]\n",
      "[{'Model #': 2, 'average': 0.6709448781975818, 'accuracy': 0.6284, 'roc_auc_score': 0.6967035421557706, 'f1 score': 0.6877310924369747}, {'Model #': 2, 'average': 0.6709448781975818, 'accuracy': 0.6284, 'roc_auc_score': 0.6967035421557706, 'f1_score': 0.6877310924369747}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1 score': 1.0}, {'Model #': 2, 'average': 1.0, 'accuracy': 1.0, 'roc_auc_score': 1.0, 'f1_score': 1.0}]\n",
      "[{'Model #': 3, 'average': 0.74206384576397, 'accuracy': 0.7158, 'roc_auc_score': 0.7922800514756035, 'f1 score': 0.7181114858163063}, {'Model #': 3, 'average': 0.7420638457639699, 'accuracy': 0.7158, 'roc_auc_score': 0.7922800514756035, 'f1_score': 0.7181114858163063}, {'Model #': 3, 'average': 0.7295420633943038, 'accuracy': 0.7058, 'roc_auc_score': 0.7799378847900615, 'f1 score': 0.7028883053928499}, {'Model #': 3, 'average': 0.7295420633943038, 'accuracy': 0.7058, 'roc_auc_score': 0.7799378847900615, 'f1_score': 0.7028883053928499}, {'Model #': 3, 'average': 0.7206933964368417, 'accuracy': 0.6992, 'roc_auc_score': 0.7648882214390389, 'f1 score': 0.697991967871486}, {'Model #': 3, 'average': 0.7206933964368417, 'accuracy': 0.6992, 'roc_auc_score': 0.7648882214390389, 'f1_score': 0.697991967871486}, {'Model #': 3, 'average': 0.7296980079045815, 'accuracy': 0.7068, 'roc_auc_score': 0.7843129771542059, 'f1 score': 0.6979810465595385}, {'Model #': 3, 'average': 0.7296980079045814, 'accuracy': 0.7068, 'roc_auc_score': 0.7843129771542059, 'f1_score': 0.6979810465595385}, {'Model #': 3, 'average': 0.7606319834637382, 'accuracy': 0.7332, 'roc_auc_score': 0.8176475632944405, 'f1 score': 0.7310483870967741}, {'Model #': 3, 'average': 0.7606319834637382, 'accuracy': 0.7332, 'roc_auc_score': 0.8176475632944405, 'f1_score': 0.7310483870967741}]\n",
      "[{'Model #': 4, 'average': 0.6485412946540371, 'accuracy': 0.6282, 'roc_auc_score': 0.6751302530211838, 'f1 score': 0.6422936309409274}, {'Model #': 4, 'average': 0.6485412946540371, 'accuracy': 0.6282, 'roc_auc_score': 0.6751302530211838, 'f1_score': 0.6422936309409274}, {'Model #': 4, 'average': 0.621709304049512, 'accuracy': 0.599, 'roc_auc_score': 0.6315370610459298, 'f1 score': 0.6345908511026063}, {'Model #': 4, 'average': 0.621709304049512, 'accuracy': 0.599, 'roc_auc_score': 0.6315370610459298, 'f1_score': 0.6345908511026063}, {'Model #': 4, 'average': 0.6229264883660783, 'accuracy': 0.5864, 'roc_auc_score': 0.6273911434432465, 'f1 score': 0.6549883216549883}, {'Model #': 4, 'average': 0.6229264883660783, 'accuracy': 0.5864, 'roc_auc_score': 0.6273911434432465, 'f1_score': 0.6549883216549883}, {'Model #': 4, 'average': 0.5847138914489138, 'accuracy': 0.5942, 'roc_auc_score': 0.6390443898130933, 'f1 score': 0.5208972845336481}, {'Model #': 4, 'average': 0.5847138914489138, 'accuracy': 0.5942, 'roc_auc_score': 0.6390443898130933, 'f1_score': 0.5208972845336481}, {'Model #': 4, 'average': 0.6176748467700118, 'accuracy': 0.5864, 'roc_auc_score': 0.621706957892453, 'f1 score': 0.6449175824175823}, {'Model #': 4, 'average': 0.6176748467700118, 'accuracy': 0.5864, 'roc_auc_score': 0.621706957892453, 'f1_score': 0.6449175824175823}]\n",
      "[{'Model #': 5, 'average': 0.7072981889433686, 'accuracy': 0.6814, 'roc_auc_score': 0.7490361124431215, 'f1 score': 0.6914584543869843}, {'Model #': 5, 'average': 0.7072981889433686, 'accuracy': 0.6814, 'roc_auc_score': 0.7490361124431215, 'f1_score': 0.6914584543869843}, {'Model #': 5, 'average': 0.7110489140544232, 'accuracy': 0.6858, 'roc_auc_score': 0.7442660390825663, 'f1 score': 0.7030807030807031}, {'Model #': 5, 'average': 0.7110489140544232, 'accuracy': 0.6858, 'roc_auc_score': 0.7442660390825663, 'f1_score': 0.7030807030807031}, {'Model #': 5, 'average': 0.69320483576881, 'accuracy': 0.6374, 'roc_auc_score': 0.7274119274731756, 'f1 score': 0.7148025798332546}, {'Model #': 5, 'average': 0.69320483576881, 'accuracy': 0.6374, 'roc_auc_score': 0.7274119274731756, 'f1_score': 0.7148025798332546}, {'Model #': 5, 'average': 0.7316799684696855, 'accuracy': 0.7096, 'roc_auc_score': 0.7809222008913519, 'f1 score': 0.7045177045177046}, {'Model #': 5, 'average': 0.7316799684696855, 'accuracy': 0.7096, 'roc_auc_score': 0.7809222008913519, 'f1_score': 0.7045177045177046}, {'Model #': 5, 'average': 0.6572762499863596, 'accuracy': 0.6324, 'roc_auc_score': 0.6671748269918892, 'f1 score': 0.6722539229671897}, {'Model #': 5, 'average': 0.6572762499863596, 'accuracy': 0.6324, 'roc_auc_score': 0.6671748269918892, 'f1_score': 0.6722539229671897}]\n",
      "\n",
      "set of averages of algorithms over 5 trials:\n",
      "{'1': [0.7216360618212688, 0.725602211140024, 0.6982611604798219, 0.7494914597389917, 0.7561565748411576], '2': [0.6472725902848482, 0.633537417952973, 0.6193329486231925, 0.6450118701002678, 0.6618301953002318], '3': [0.7225491368446826, 0.7246900498659006, 0.6791173744342903, 0.7250845238499908, 0.7288081064743643], '4': [0.6415731580835212, 0.61816618165116, 0.5956137193263026, 0.5684022811113622, 0.6131959825460319], '5': [0.6788812159815212, 0.6634056600353703, 0.6545530031986111, 0.6889371606935969, 0.659882205570595]}\n",
      "\n",
      "set of acc values of algorithms over 5 trials\n",
      "{'1': [0.7015, 0.7055, 0.6765, 0.734, 0.735], '2': [0.6115, 0.604, 0.5845, 0.6155, 0.627], '3': [0.7, 0.7015, 0.66, 0.705, 0.705], '4': [0.62, 0.597, 0.5655, 0.5825, 0.5805], '5': [0.652, 0.6425, 0.6115, 0.671, 0.6375]}\n",
      "\n",
      "set of roc values of algorithms over 5 trials\n",
      "{'1': [0.7690917492118866, 0.7763299374076432, 0.752650406504065, 0.7953613908326033, 0.8038778877887788], '2': [0.6602540765870285, 0.6667057118028441, 0.6420132582864291, 0.6698772959500061, 0.6856835683568356], '3': [0.7646771135043446, 0.7721185690658401, 0.7200940587867417, 0.7734395941604555, 0.7714931493149316], '4': [0.668008193179818, 0.6341340589721718, 0.5881626016260163, 0.6242083448355878, 0.6211941194119412], '5': [0.7151279784288938, 0.6922952933591231, 0.6671164477798623, 0.7281347144040231, 0.6724427442744274]}\n",
      "\n",
      "set of f1 values of algorithms over 5 trials\n",
      "{'1': [0.6943164362519201, 0.6949766960124287, 0.6656330749354005, 0.7191129883843718, 0.7295918367346939], '2': [0.670063694267516, 0.6299065420560747, 0.6314855875831487, 0.6496583143507972, 0.6728070175438597], '3': [0.702970297029703, 0.7004515805318615, 0.657258064516129, 0.696813977389517, 0.7099311701081613], '4': [0.6367112810707457, 0.6233644859813084, 0.6331785563528916, 0.49849849849849853, 0.6378938282261546], '5': [0.6695156695156694, 0.655421686746988, 0.6850425618159709, 0.6676767676767676, 0.6697038724373576]}\n"
     ]
    }
   ],
   "source": [
    "print('Best Models On Average For Test Set:')\n",
    "for element in best_clf_list:\n",
    "    print(element)\n",
    "\n",
    "print()\n",
    "\n",
    "print('Train Set Data')\n",
    "for i in range(len(training_perf)):\n",
    "    print(training_perf[str(i + 1)])\n",
    "\n",
    "print()\n",
    "\n",
    "alg_avg = {}\n",
    "alg_acc = {}\n",
    "alg_roc = {}\n",
    "alg_f1 = {}\n",
    "\n",
    "for i in range(len(trial_storage)):\n",
    "    alg_avg[str(i + 1)]=[]\n",
    "    alg_acc[str(i + 1)]=[]\n",
    "    alg_roc[str(i + 1)]=[]\n",
    "    alg_f1[str(i + 1)]=[]\n",
    "    for entry in trial_storage[str(i + 1)]:\n",
    "        alg_avg[str(i + 1)].append(entry['average'])\n",
    "        alg_acc[str(i + 1)].append(entry['accuracy'])\n",
    "        alg_roc[str(i + 1)].append(entry['roc_auc_score'])\n",
    "        alg_f1[str(i + 1)].append(entry['f1_score'])\n",
    "\n",
    "print('set of averages of algorithms over 5 trials:')\n",
    "print(alg_avg)\n",
    "print()\n",
    "print('set of acc values of algorithms over 5 trials')\n",
    "print(alg_acc)\n",
    "print()\n",
    "print('set of roc values of algorithms over 5 trials')\n",
    "print(alg_roc)\n",
    "print()\n",
    "print('set of f1 values of algorithms over 5 trials')\n",
    "print(alg_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7104999999999999, '2': 0.6085, '3': 0.6943, '4': 0.5891, '5': 0.6429}\n"
     ]
    }
   ],
   "source": [
    "# calculate average acc metric scores per algorithm over 5 trials\n",
    "alg_acc_averages = {}\n",
    "for i in range(len(alg_acc)):\n",
    "    alg_acc_averages[str(i + 1)] = sum(alg_acc[str(i + 1)])/5\n",
    "\n",
    "print(alg_acc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7794622743489954, '2': 0.6649067821966286, '3': 0.7603644969664627, '4': 0.627141463605107, '5': 0.6950234356492659}\n"
     ]
    }
   ],
   "source": [
    "# calculate average roc metric scores per algorithm over 5 trials\n",
    "alg_roc_averages = {}\n",
    "for i in range(len(alg_roc)):\n",
    "    alg_roc_averages[str(i + 1)] = sum(alg_roc[str(i + 1)])/5\n",
    "\n",
    "print(alg_roc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7007262064637629, '2': 0.6507842311602793, '3': 0.6934850179150744, '4': 0.6059293300259198, '5': 0.6694721116385507}\n"
     ]
    }
   ],
   "source": [
    "# calculate average f1 metric scores per algorithm over 5 trials\n",
    "alg_f1_averages = {}\n",
    "for i in range(len(alg_f1)):\n",
    "    alg_f1_averages[str(i + 1)] = sum(alg_f1[str(i + 1)])/5\n",
    "\n",
    "print(alg_f1_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7302294936042527, '2': 0.6413970044523026, '3': 0.7160498382938458, '4': 0.6073902645436755, '5': 0.6691318490959389}\n"
     ]
    }
   ],
   "source": [
    "averages = {}\n",
    "for i in range(len(alg_acc_averages)):\n",
    "    averages[str(i + 1)] = (alg_acc_averages[str(i + 1)] + alg_roc_averages[str(i + 1)] + alg_f1_averages[str(i + 1)])/3\n",
    "\n",
    "print(averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ttest_relResult(statistic=10.85608253750064, pvalue=3.3462524215480636e-08)\n",
      "Ttest_relResult(statistic=3.9539094923767744, pvalue=0.0014406524366184668)\n",
      "Ttest_relResult(statistic=9.13710780210442, pvalue=2.822386600659466e-07)\n",
      "Ttest_relResult(statistic=6.999851511777664, pvalue=6.249422937916668e-06)\n"
     ]
    }
   ],
   "source": [
    "# t-test best against rest mean of metrics (RF against rest)\n",
    "combined_metrics_1 = []\n",
    "combined_metrics_2 = []\n",
    "combined_metrics_3 = []\n",
    "combined_metrics_4 = []\n",
    "combined_metrics_5 = []\n",
    "\n",
    "for item in alg_acc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_roc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_f1['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "    \n",
    "for item in alg_acc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_roc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_f1['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "\n",
    "for item in alg_acc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_roc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_f1['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "    \n",
    "for item in alg_acc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_roc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_f1['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "\n",
    "for item in alg_acc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_roc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_f1['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "    \n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_2))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_3))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_4))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}