applications/Parkinson's Disease classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The dataset** is collected from UCI Machine Learning Repository through the following [link](https://archive.ics.uci.edu/ml/datasets/Parkinson%27s+Disease+Classification#)\n",
    "\n",
    "extract data with its default name `pd_speech_features.csv` in `__data__` directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>PPE</th>\n",
       "      <th>DFA</th>\n",
       "      <th>RPDE</th>\n",
       "      <th>numPulses</th>\n",
       "      <th>numPeriodsPulses</th>\n",
       "      <th>meanPeriodPulses</th>\n",
       "      <th>stdDevPeriodPulses</th>\n",
       "      <th>locPctJitter</th>\n",
       "      <th>...</th>\n",
       "      <th>tqwt_kurtosisValue_dec_28</th>\n",
       "      <th>tqwt_kurtosisValue_dec_29</th>\n",
       "      <th>tqwt_kurtosisValue_dec_30</th>\n",
       "      <th>tqwt_kurtosisValue_dec_31</th>\n",
       "      <th>tqwt_kurtosisValue_dec_32</th>\n",
       "      <th>tqwt_kurtosisValue_dec_33</th>\n",
       "      <th>tqwt_kurtosisValue_dec_34</th>\n",
       "      <th>tqwt_kurtosisValue_dec_35</th>\n",
       "      <th>tqwt_kurtosisValue_dec_36</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.85247</td>\n",
       "      <td>0.71826</td>\n",
       "      <td>0.57227</td>\n",
       "      <td>240</td>\n",
       "      <td>239</td>\n",
       "      <td>0.008064</td>\n",
       "      <td>0.000087</td>\n",
       "      <td>0.00218</td>\n",
       "      <td>...</td>\n",
       "      <td>1.5620</td>\n",
       "      <td>2.6445</td>\n",
       "      <td>3.8686</td>\n",
       "      <td>4.2105</td>\n",
       "      <td>5.1221</td>\n",
       "      <td>4.4625</td>\n",
       "      <td>2.6202</td>\n",
       "      <td>3.0004</td>\n",
       "      <td>18.9405</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.76686</td>\n",
       "      <td>0.69481</td>\n",
       "      <td>0.53966</td>\n",
       "      <td>234</td>\n",
       "      <td>233</td>\n",
       "      <td>0.008258</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>0.00195</td>\n",
       "      <td>...</td>\n",
       "      <td>1.5589</td>\n",
       "      <td>3.6107</td>\n",
       "      <td>23.5155</td>\n",
       "      <td>14.1962</td>\n",
       "      <td>11.0261</td>\n",
       "      <td>9.5082</td>\n",
       "      <td>6.5245</td>\n",
       "      <td>6.3431</td>\n",
       "      <td>45.1780</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.85083</td>\n",
       "      <td>0.67604</td>\n",
       "      <td>0.58982</td>\n",
       "      <td>232</td>\n",
       "      <td>231</td>\n",
       "      <td>0.008340</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>0.00176</td>\n",
       "      <td>...</td>\n",
       "      <td>1.5643</td>\n",
       "      <td>2.3308</td>\n",
       "      <td>9.4959</td>\n",
       "      <td>10.7458</td>\n",
       "      <td>11.0177</td>\n",
       "      <td>4.8066</td>\n",
       "      <td>2.9199</td>\n",
       "      <td>3.1495</td>\n",
       "      <td>4.7666</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.41121</td>\n",
       "      <td>0.79672</td>\n",
       "      <td>0.59257</td>\n",
       "      <td>178</td>\n",
       "      <td>177</td>\n",
       "      <td>0.010858</td>\n",
       "      <td>0.000183</td>\n",
       "      <td>0.00419</td>\n",
       "      <td>...</td>\n",
       "      <td>3.7805</td>\n",
       "      <td>3.5664</td>\n",
       "      <td>5.2558</td>\n",
       "      <td>14.0403</td>\n",
       "      <td>4.2235</td>\n",
       "      <td>4.6857</td>\n",
       "      <td>4.8460</td>\n",
       "      <td>6.2650</td>\n",
       "      <td>4.0603</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.32790</td>\n",
       "      <td>0.79782</td>\n",
       "      <td>0.53028</td>\n",
       "      <td>236</td>\n",
       "      <td>235</td>\n",
       "      <td>0.008162</td>\n",
       "      <td>0.002669</td>\n",
       "      <td>0.00535</td>\n",
       "      <td>...</td>\n",
       "      <td>6.1727</td>\n",
       "      <td>5.8416</td>\n",
       "      <td>6.0805</td>\n",
       "      <td>5.7621</td>\n",
       "      <td>7.7817</td>\n",
       "      <td>11.6891</td>\n",
       "      <td>8.2103</td>\n",
       "      <td>5.0559</td>\n",
       "      <td>6.1164</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 755 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  gender      PPE      DFA     RPDE  numPulses  numPeriodsPulses  \\\n",
       "0   0       1  0.85247  0.71826  0.57227        240               239   \n",
       "1   0       1  0.76686  0.69481  0.53966        234               233   \n",
       "2   0       1  0.85083  0.67604  0.58982        232               231   \n",
       "3   1       0  0.41121  0.79672  0.59257        178               177   \n",
       "4   1       0  0.32790  0.79782  0.53028        236               235   \n",
       "\n",
       "   meanPeriodPulses  stdDevPeriodPulses  locPctJitter  ...  \\\n",
       "0          0.008064            0.000087       0.00218  ...   \n",
       "1          0.008258            0.000073       0.00195  ...   \n",
       "2          0.008340            0.000060       0.00176  ...   \n",
       "3          0.010858            0.000183       0.00419  ...   \n",
       "4          0.008162            0.002669       0.00535  ...   \n",
       "\n",
       "   tqwt_kurtosisValue_dec_28  tqwt_kurtosisValue_dec_29  \\\n",
       "0                     1.5620                     2.6445   \n",
       "1                     1.5589                     3.6107   \n",
       "2                     1.5643                     2.3308   \n",
       "3                     3.7805                     3.5664   \n",
       "4                     6.1727                     5.8416   \n",
       "\n",
       "   tqwt_kurtosisValue_dec_30  tqwt_kurtosisValue_dec_31  \\\n",
       "0                     3.8686                     4.2105   \n",
       "1                    23.5155                    14.1962   \n",
       "2                     9.4959                    10.7458   \n",
       "3                     5.2558                    14.0403   \n",
       "4                     6.0805                     5.7621   \n",
       "\n",
       "   tqwt_kurtosisValue_dec_32  tqwt_kurtosisValue_dec_33  \\\n",
       "0                     5.1221                     4.4625   \n",
       "1                    11.0261                     9.5082   \n",
       "2                    11.0177                     4.8066   \n",
       "3                     4.2235                     4.6857   \n",
       "4                     7.7817                    11.6891   \n",
       "\n",
       "   tqwt_kurtosisValue_dec_34  tqwt_kurtosisValue_dec_35  \\\n",
       "0                     2.6202                     3.0004   \n",
       "1                     6.5245                     6.3431   \n",
       "2                     2.9199                     3.1495   \n",
       "3                     4.8460                     6.2650   \n",
       "4                     8.2103                     5.0559   \n",
       "\n",
       "   tqwt_kurtosisValue_dec_36  class  \n",
       "0                    18.9405      1  \n",
       "1                    45.1780      1  \n",
       "2                     4.7666      1  \n",
       "3                     4.0603      1  \n",
       "4                     6.1164      1  \n",
       "\n",
       "[5 rows x 755 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./__data__/pd_speech_features.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_random_seed(seed=1917):\n",
    "    np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.copy()\n",
    "y = X.pop('class')\n",
    "ids = X.pop('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to panda dataframes to numpy nd-arrays\n",
    "X = X.to_numpy()\n",
    "y = y.values\n",
    "ids = ids.values\n",
    "unique_id = np.unique(ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit & inform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict \n",
    "from scipy.stats import mode\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score, \n",
    "    precision_score, \n",
    "    recall_score, \n",
    "    f1_score, \n",
    "    matthews_corrcoef\n",
    ")\n",
    "\n",
    "def fit_inform(model, model_data, metrics=None):\n",
    "    if (metrics is None): metrics = defaultdict(list)\n",
    "    (X_train, X_test, y_train, y_test) = model_data\n",
    "\n",
    "    # fit models\n",
    "    model.fit(X_train, y_train)\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    # majority vote\n",
    "    test_index_ids = ids[test_index]\n",
    "    for test_id in unique_test_ids:\n",
    "        y_indexs = test_index_ids == test_id \n",
    "        y_pred[y_indexs] = mode(y_pred[y_indexs]).mode[0]\n",
    "\n",
    "    y_true = y_test\n",
    "\n",
    "    metrics[\"accuracy\"].append(accuracy_score(y_true, y_pred))\n",
    "    metrics[\"precision\"].append(precision_score(y_true, y_pred))\n",
    "    metrics[\"recall\"].append(recall_score(y_true, y_pred))\n",
    "    metrics[\"fMeasure\"].append(f1_score(y_true, y_pred))\n",
    "    metrics[\"mcc\"].append(matthews_corrcoef(y_true, y_pred))\n",
    "\n",
    "\n",
    "    for metric in metrics:\n",
    "        cur_metric = metrics[metric]  \n",
    "        if metric == 'accuracy':\n",
    "            metrics[metric] = f\"N({np.mean(cur_metric):.3}, {np.std(cur_metric):.2})\"\n",
    "        else:\n",
    "            metrics[metric] = f\"{np.mean(cur_metric):.3}\"\n",
    "            \n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_name(model):\n",
    "    return type(model).__name__\n",
    "def get_hyper_parameter(model):\n",
    "    return { key:value for (key, value) in model.get_params().items() if value }\n",
    "\n",
    "def add_extra_metrics(metrics, model, pca):\n",
    "    metrics[get_model_name(model)] = get_hyper_parameter(model)\n",
    "    metrics['PCA'] = get_hyper_parameter(pca)\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "unique_train_ids, unique_test_ids = train_test_split(unique_id, test_size=0.3)\n",
    "\n",
    "# same person same predict\n",
    "train_index = np.isin(ids, unique_train_ids)\n",
    "test_index = np.isin(ids, unique_test_ids)\n",
    "\n",
    "# test and train data\n",
    "X_train, X_test = X[train_index], X[test_index]\n",
    "y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "# PCA min(n_samples, n_features)=528\n",
    "pca = PCA(n_components=100)\n",
    "X_train = pca.fit_transform(X_train)\n",
    "X_test = pca.transform(X_test)\n",
    "\n",
    "model_data = (X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'list'>,\n",
      "            {'DecisionTreeClassifier': {'criterion': 'entropy',\n",
      "                                        'min_samples_leaf': 1,\n",
      "                                        'min_samples_split': 5,\n",
      "                                        'min_weight_fraction_leaf': 0.1,\n",
      "                                        'presort': 'deprecated',\n",
      "                                        'splitter': 'best'},\n",
      "             'PCA': {'copy': True,\n",
      "                     'iterated_power': 'auto',\n",
      "                     'n_components': 100,\n",
      "                     'svd_solver': 'auto'},\n",
      "             'accuracy': 'N(0.763, 0.0)',\n",
      "             'fMeasure': '0.85',\n",
      "             'mcc': '0.331',\n",
      "             'precision': '0.785',\n",
      "             'recall': '0.927'})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Decision Tree \"\"\"\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "reset_random_seed()\n",
    "\n",
    "model = DecisionTreeClassifier(min_weight_fraction_leaf=0.1, criterion=\"entropy\", min_samples_split=5)\n",
    "metrics = fit_inform(model, model_data)\n",
    "metrics = add_extra_metrics(metrics, model, pca)\n",
    "pprint(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'list'>,\n",
      "            {'PCA': {'copy': True,\n",
      "                     'iterated_power': 'auto',\n",
      "                     'n_components': 100,\n",
      "                     'svd_solver': 'auto'},\n",
      "             'RandomForestClassifier': {'criterion': 'gini',\n",
      "                                        'max_features': 'auto',\n",
      "                                        'min_samples_leaf': 1,\n",
      "                                        'min_samples_split': 2,\n",
      "                                        'n_estimators': 90},\n",
      "             'accuracy': 'N(0.776, 0.0)',\n",
      "             'fMeasure': '0.866',\n",
      "             'mcc': '0.227',\n",
      "             'precision': '0.809',\n",
      "             'recall': '0.932'})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" RandomForestClassifier \"\"\"\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "reset_random_seed()\n",
    "model = RandomForestClassifier(bootstrap=False, n_estimators=90)\n",
    "metrics = fit_inform(model, model_data)\n",
    "metrics = add_extra_metrics(metrics, model, pca)\n",
    "pprint(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'list'>,\n",
      "            {'GradientBoostingClassifier': {'criterion': 'friedman_mse',\n",
      "                                            'learning_rate': 0.1,\n",
      "                                            'loss': 'deviance',\n",
      "                                            'max_depth': 3,\n",
      "                                            'min_samples_leaf': 1,\n",
      "                                            'min_samples_split': 2,\n",
      "                                            'n_estimators': 105,\n",
      "                                            'presort': 'deprecated',\n",
      "                                            'subsample': 1.0,\n",
      "                                            'tol': 0.0001,\n",
      "                                            'validation_fraction': 0.1},\n",
      "             'PCA': {'copy': True,\n",
      "                     'iterated_power': 'auto',\n",
      "                     'n_components': 100,\n",
      "                     'svd_solver': 'auto'},\n",
      "             'accuracy': 'N(0.842, 0.0)',\n",
      "             'fMeasure': '0.9',\n",
      "             'mcc': '0.527',\n",
      "             'precision': '0.885',\n",
      "             'recall': '0.915'})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" XGBoost (GradientBoostingClassifier) \"\"\"\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "reset_random_seed()\n",
    "model = GradientBoostingClassifier(n_estimators=105)\n",
    "metrics = fit_inform(model, model_data)\n",
    "metrics = add_extra_metrics(metrics, model, pca)\n",
    "pprint(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'N(0.776, 0.0)'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    }
   ],
   "source": [
    "\"\"\" SVM \"\"\"\n",
    "from sklearn.svm import SVC as SVM\n",
    "reset_random_seed()\n",
    "model = SVM(kernel=\"poly\", degree=1)\n",
    "metrics = fit_inform(model, model_data)\n",
    "metrics = add_extra_metrics(metrics, model, pca)\n",
    "pprint(metrics['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "def cross_validation(model, X, y, k=5):\n",
    "    metrics = defaultdict(list)\n",
    "    kf = KFold(n_splits=k)\n",
    "    \n",
    "    for unique_train_ids, unique_test_ids in kf.split(unique_id):\n",
    "        # same person same predict\n",
    "        train_index = np.isin(ids, unique_train_ids)\n",
    "        test_index = np.isin(ids, unique_test_ids)\n",
    "\n",
    "        # test and train data\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "        # PCA min(n_samples, n_features)=528\n",
    "        pca = PCA(n_components=100)\n",
    "        X_train = pca.fit_transform(X_train)\n",
    "        X_test = pca.transform(X_test)\n",
    "        \n",
    "        # fit models\n",
    "        model.fit(X_train, y_train)\n",
    "        # predict\n",
    "        y_pred = model.predict(X_test)\n",
    "        \n",
    "        # majority vote\n",
    "        test_index_ids = ids[test_index]\n",
    "        for test_id in unique_test_ids:\n",
    "            y_indexs = test_index_ids == test_id \n",
    "            y_pred[y_indexs] = mode(y_pred[y_indexs]).mode[0]\n",
    "\n",
    "        y_true = y_test\n",
    "\n",
    "        metrics[\"accuracy\"].append(accuracy_score(y_true, y_pred))\n",
    "        metrics[\"precision\"].append(precision_score(y_true, y_pred))\n",
    "        metrics[\"recall\"].append(recall_score(y_true, y_pred))\n",
    "        metrics[\"fMeasure\"].append(f1_score(y_true, y_pred))\n",
    "        metrics[\"mcc\"].append(matthews_corrcoef(y_true, y_pred))\n",
    "\n",
    "\n",
    "    for metric in metrics:\n",
    "        cur_metric = metrics[metric]  \n",
    "        if metric == 'accuracy':\n",
    "            metrics[metric] = f\"N({np.mean(cur_metric):.3}, {np.std(cur_metric):.2})\"\n",
    "        else:\n",
    "            metrics[metric] = f\"{np.mean(cur_metric):.3}\"\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'DecisionTreeClassifier': defaultdict(<class 'list'>,\n",
      "                                       {'accuracy': 'N(0.742, 0.088)',\n",
      "                                        'fMeasure': '0.838',\n",
      "                                        'mcc': '0.208',\n",
      "                                        'precision': '0.773',\n",
      "                                        'recall': '0.916'})}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/amirhossein/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:900: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'SVM': defaultdict(<class 'list'>,\n",
      "                    {'accuracy': 'N(0.746, 0.062)',\n",
      "                     'fMeasure': '0.853',\n",
      "                     'mcc': '0.0',\n",
      "                     'precision': '0.746',\n",
      "                     'recall': '1.0'})}\n",
      "{'GradientBoostingClassifier': 'N(0.806, 0.029)'}\n",
      "{'RandomForestClassifier': defaultdict(<class 'list'>,\n",
      "                                       {'accuracy': 'N(0.794, 0.054)',\n",
      "                                        'fMeasure': '0.872',\n",
      "                                        'mcc': '0.371',\n",
      "                                        'precision': '0.803',\n",
      "                                        'recall': '0.956'})}\n"
     ]
    }
   ],
   "source": [
    "reset_random_seed()\n",
    "model = DecisionTreeClassifier(min_weight_fraction_leaf=0.1, criterion=\"entropy\", min_samples_split=5)\n",
    "pprint({'DecisionTreeClassifier': cross_validation(model, X, y)})\n",
    "\n",
    "reset_random_seed()\n",
    "model = SVM(kernel=\"poly\", degree=1)\n",
    "pprint({'SVM': cross_validation(model, X, y)})\n",
    "\n",
    "reset_random_seed()\n",
    "model = GradientBoostingClassifier(subsample=0.84, n_estimators=125, min_samples_split=20, max_features='log2')\n",
    "pprint({'GradientBoostingClassifier': cross_validation(model, X, y)['accuracy']})\n",
    "\n",
    "reset_random_seed()\n",
    "model = RandomForestClassifier(bootstrap=False, n_estimators=90)\n",
    "pprint({'RandomForestClassifier': cross_validation(model, X, y)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|method          | accuracy | f-measure | percision | recall | MCC |\n",
    "|:------         |:--------:|:---------:|:---------:|:------:|:---:|\n",
    "|SVM             | 0.732    | 0.841     | 0.751     | 0.959  |  -  |\n",
    "|SVC          (T)| 0.746    | 0.853     | 0.746     | 1.0    |0.0  |\n",
    "|Decision tree   | 0.720    | 0.815     | 0.808     | 0.828  |  -  |\n",
    "|Decision tree(T)| 0.791    | 0.866     | 0.816     | 0.924  |0.358|\n",
    "|Random Forest   | 0.832    | 0.892     | 0.842     | 0.951  |  -  |\n",
    "|Random Forest(T)| 0.839    | 0.897     | 0.847     | 0.956  |0.521|\n",
    "|XGBoost         | 0.841    | 0.896     | 0.857     | 0.939  |  -  |\n",
    "|XGBoost      (T)|**0.86**  | 0.909     | 0.862     | 0.963  |0.598|\n",
    "|[paper][link]   | 0.86     | 0.84      |   -       |    -   | 0.59|\n",
    "|SVC          (PT)| 0.746    | 0.853     | 0.746     | 1.0    |0.0  |\n",
    "|Decision tree(PT)| 0.746    | 0.84     | 0.776     | 0.916  |0.224|\n",
    "|Random Forest(PT)| 0.781    | 0.866     | 0.790     | 0.961  |0.322|\n",
    "|XGBoost      (PT)| 0.802    | 0.875     | 0.828     | 0.932  |0.39|\n",
    "\n",
    "> (T) means hyper parameter tuned in this version (5-fold)   \n",
    "> (PT) paper method with hyper parameter tuned (person aggregation + majority vote + 5-fold)\n",
    "\n",
    "\n",
    "[link]: https://www.sciencedirect.com/science/article/abs/pii/S1568494618305799?via%3Dihub"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Old Method with correlation removal\n",
    "\n",
    "\n",
    "### Preprocessing Data\n",
    "- Remove nearly the same data\n",
    "pearson correlation provided by [pandas](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.corr.html) is used\n",
    "\n",
    "<p align=\"cetner\">\n",
    "    <img src=\"https://github.com/amirHossein-Ebrahimi/leaf-node/raw/671c396db2f81a2352941ec572dd45b04252b7da/applications/doc/images/correlation.jpg\">\n",
    "</p>\n",
    "\n",
    "<sub>For high quality image, visit [link](https://render.githubusercontent.com/view/kaggle_corr)</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "app_LT_entropy_logs = [att for att in X.columns.values if att.startswith('app_LT_entropy_log')]\n",
    "sns.heatmap(X[app_LT_entropy_logs].corr(), annot=False)\n",
    "\n",
    "selected_feature = app_LT_entropy_logs[:5]\n",
    "X['app_LT_entropy_logs[:5]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "selected_feature = app_LT_entropy_logs[5:7]\n",
    "X['app_LT_entropy_logs[5:7]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "selected_feature = app_LT_entropy_logs[7:]\n",
    "X['app_LT_entropy_logs[7:]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "category = 'app_det_TKEO_mean'\n",
    "selected_feature = [att for att in X.columns.values if att.startswith(category)]\n",
    "selected_feature = selected_feature[3:]\n",
    "X[f'{category}[3:]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "category = 'app_TKEO_std'\n",
    "selected_feature = [att for att in X.columns.values if att.startswith(category)]\n",
    "selected_feature = selected_feature[3:]\n",
    "X[f'{category}[3:]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "category = 'app_LT_TKEO_mean'\n",
    "selected_feature = [att for att in X.columns.values if att.startswith(category)]\n",
    "selected_feature = selected_feature[4:]\n",
    "X[f'{category}[4:]'] = X[selected_feature].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature, axis=1, inplace=True)\n",
    "\n",
    "category = 'app_LT_TKEO_std'\n",
    "selected_feature = [att for att in X.columns.values if att.startswith(category)]\n",
    "X[f'{category}[3:6]'] = X[selected_feature[3:6]].mean(numeric_only=True, axis=1)\n",
    "X[f'{category}[6:]'] = X[selected_feature[6:]].mean(numeric_only=True, axis=1)\n",
    "X.drop(selected_feature[3:], axis=1, inplace=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}