combustion-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import scipy as sc\n",
    "import cv2 as cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.io as scio\n",
    "import h5py\n",
    "from skimage.feature import hog\n",
    "from skimage import data, exposure\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import svm\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.decomposition import MiniBatchDictionaryLearning\n",
    "from time import time\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract data\n",
    "#comb_data = h5py.File('../Aditya_data/combustion_img_13.mat','r')\n",
    "comb_data = h5py.File('combustion_img_13.mat','r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = comb_data['train_set_x'][()]\n",
    "y_train = comb_data['train_set_y'][()]\n",
    "y_train = np.ravel(y_train)\n",
    "X_test = comb_data['test_set_x'][()]\n",
    "y_test = comb_data['test_set_y'][()]\n",
    "y_test = np.ravel(y_test)\n",
    "X_valid = comb_data['valid_set_x'][()]\n",
    "y_valid = comb_data['valid_set_y'][()]\n",
    "y_valid = np.ravel(y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54000,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_final = np.zeros([54000,1250])\n",
    "n=len(X_train_final[:,1])\n",
    "np.shape(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Feature extraction using Histogram of Gradients\n",
    "#Training data\n",
    "for i in range(0,n):\n",
    "    temp_image = X_train[:,i]\n",
    "    \n",
    "    temp_image = np.reshape(temp_image,[250,100])\n",
    "    temp_image = temp_image.T\n",
    "    temp_fd = hog(temp_image, orientations=5, pixels_per_cell=(10, 10),\n",
    "                    cells_per_block=(1, 1))\n",
    "    \n",
    "    X_train_final[i,:] = temp_fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract features of test data using Histogram of Gradients\n",
    "X_test_final = np.zeros([18000,1250])\n",
    "b=len(X_test_final[:,1])\n",
    "for i in range(0,b):\n",
    "    temp_image = X_test[:,i]\n",
    "    \n",
    "    temp_image = np.reshape(temp_image,[250,100])\n",
    "    temp_image = temp_image.T\n",
    "    temp_fd = hog(temp_image, orientations=5, pixels_per_cell=(10, 10),\n",
    "                    cells_per_block=(1, 1))\n",
    "    \n",
    "    X_test_final[i,:] = temp_fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Extract features of validation data using Histogram of Gradients\n",
    "X_valid_final = np.zeros([9000,1250])\n",
    "b=len(X_valid_final[:,1])\n",
    "for i in range(0,b):\n",
    "    temp_image = X_valid[:,i]\n",
    "    \n",
    "    temp_image = np.reshape(temp_image,[250,100])\n",
    "    temp_image = temp_image.T\n",
    "    temp_fd = hog(temp_image, orientations=5, pixels_per_cell=(10, 10),\n",
    "                    cells_per_block=(1, 1))\n",
    "    \n",
    "    X_valid_final[i,:] = temp_fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 54000)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning the dictionary...\n",
      "done in 139.22s.\n"
     ]
    }
   ],
   "source": [
    "#Feature extraction using Dictionary Learning\n",
    "#Training data\n",
    "print('Learning the dictionary...')\n",
    "t0 = time()\n",
    "dico = MiniBatchDictionaryLearning(n_components=10, alpha=1, n_iter=100)\n",
    "X_train_dict = dico.fit_transform(X_train.T)\n",
    "np.shape(X_train_dict)\n",
    "dt = time() - t0\n",
    "print('done in %.2fs.' % dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18000, 10)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Express test data in terms of (Dictionary) learned features\n",
    "X_test_dict = dico.transform(X_test.T)\n",
    "np.shape(X_test_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9000, 10)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Express validation data in terms of (Dictionary) learned features\n",
    "X_valid_dict = dico.transform(X_valid.T)\n",
    "np.shape(X_valid_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=4, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=2, n_jobs=1,\n",
       "            oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Random forest classifier for HOG features # n_features = 1250\n",
    "num_features = \"auto\" #default option \"auto\" = sqrt(n_features); \"log2\" = log2(n_features); None = n_features \n",
    "clf = RandomForestClassifier(n_estimators=2, max_features=32, max_depth=2, random_state=0)\n",
    "clf.fit(X_train_final, y_train)\n",
    "\n",
    "#Random forest classifier for dictionary features # n_features = 10\n",
    "num_features_d = \"auto\" #default option \"auto\" = sqrt(n_features); \"log2\" = log2(n_features); None = n_features \n",
    "clfd = RandomForestClassifier(n_estimators=2, max_features=num_features_d, max_depth=4, random_state=0)\n",
    "clfd.fit(X_train_dict, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=4, max_features=1250, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=6, n_jobs=1,\n",
       "            oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Random forest classifier for HOG features # n_features = 1250\n",
    "num_features = \"auto\" #default option \"auto\" = sqrt(n_features); \"log2\" = log2(n_features); None = n_features \n",
    "clf = RandomForestClassifier(n_estimators=6, max_features=1250, max_depth=4, random_state=0)\n",
    "clf.fit(X_train_final, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Random Forest Classifier using HOG features is... 0.971388888889\n"
     ]
    }
   ],
   "source": [
    "#Test classification accuracy using Random Forest Classifier for HOG features\n",
    "\n",
    "y_test_predict=clf.predict(X_test_final)\n",
    "acc = accuracy_score(y_test, y_test_predict)\n",
    "print('Accuracy of Random Forest Classifier using HOG features is...',acc)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=4, max_features=10, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=6, n_jobs=1,\n",
       "            oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Random forest classifier for dictionary features # n_features = 10\n",
    "num_features_d = \"auto\" #default option \"auto\" = sqrt(n_features); \"log2\" = log2(n_features); None = n_features \n",
    "clfd = RandomForestClassifier(n_estimators=6, max_features=10, max_depth=4, random_state=0)\n",
    "clfd.fit(X_train_dict, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Random Forest Classifier using Dictionary learned features is... 0.914333333333\n"
     ]
    }
   ],
   "source": [
    "#Test classification accuracy using Random Forest Classifier for dictionary features\n",
    "\n",
    "y_test_predictd=clfd.predict(X_test_dict)\n",
    "acc_d = accuracy_score(y_test, y_test_predictd)\n",
    "print('Accuracy of Random Forest Classifier using Dictionary learned features is...',acc_d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Decision Tree Classifier using HOG features is... 0.915833333333\n",
      "Accuracy of Decision Tree Classifier using Dictionary learned features is... 0.925111111111\n"
     ]
    }
   ],
   "source": [
    "##Compare the performance to Decision Tree, Extra Trees, AdaBoost, Voting and Gradient Tree Boosting.\n",
    "\n",
    "#Decision Tree\n",
    "clf_dt = tree.DecisionTreeClassifier(max_depth=4, max_features=30)\n",
    "clf_dt.fit(X_train_final, y_train)\n",
    "y_test_predict_dectree=clf_dt.predict(X_test_final)\n",
    "acc_dt = accuracy_score(y_test, y_test_predict_dectree)\n",
    "print('Accuracy of Decision Tree Classifier using HOG features is...',acc_dt)\n",
    "clf_dt_d = tree.DecisionTreeClassifier(max_depth=4, max_features=9)\n",
    "clf_dt_d.fit(X_train_dict, y_train)\n",
    "y_test_predict_dectree_d=clf_dt_d.predict(X_test_dict)\n",
    "acc_dt_d = accuracy_score(y_test, y_test_predict_dectree_d)\n",
    "print('Accuracy of Decision Tree Classifier using Dictionary learned features is...',acc_dt_d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Extra Trees Classifier using HOG features is... 0.889\n",
      "Accuracy of Extra Trees Classifier using Dictionary learned features is... 0.924055555556\n"
     ]
    }
   ],
   "source": [
    "#Extra Trees\n",
    "extclf= ExtraTreesClassifier(n_estimators=250, max_depth=4, max_features=30, random_state=0)\n",
    "extclf.fit(X_train_final, y_train)\n",
    "y_test_predict_exttree=extclf.predict(X_test_final)\n",
    "acc_et = accuracy_score(y_test, y_test_predict_exttree)\n",
    "print('Accuracy of Extra Trees Classifier using HOG features is...',acc_et)\n",
    "extclf_d= ExtraTreesClassifier(n_estimators=10, max_depth=4, max_features=9, random_state=0)\n",
    "extclf_d.fit(X_train_dict, y_train)\n",
    "y_test_predict_exttree_d=extclf_d.predict(X_test_dict)\n",
    "acc_et_d = accuracy_score(y_test, y_test_predict_exttree_d)\n",
    "print('Accuracy of Extra Trees Classifier using Dictionary learned features is...',acc_et_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Gradient Boosting Classifier using HOG features is... 0.996111111111\n",
      "Accuracy of Gradient Boosting Classifier using Dictionary learned features is... 0.99\n"
     ]
    }
   ],
   "source": [
    "#Gradient Boosting Classifier\n",
    "clfgb = GradientBoostingClassifier(n_estimators=30, learning_rate=1.0, max_depth=4, max_features=3, random_state=0)\n",
    "clfgb.fit(X_train_final, y_train)\n",
    "y_test_predict_gradboost=clfgb.predict(X_test_final)\n",
    "acc_gb = accuracy_score(y_test, y_test_predict_gradboost)\n",
    "print('Accuracy of Gradient Boosting Classifier using HOG features is...',acc_gb)\n",
    "clfgb_d = GradientBoostingClassifier(n_estimators=9, learning_rate=1.0, max_depth=4, max_features=3, random_state=0)\n",
    "clfgb_d.fit(X_train_dict, y_train)\n",
    "y_test_predict_gradboostd=clfgb_d.predict(X_test_dict)\n",
    "acc_gb_d = accuracy_score(y_test, y_test_predict_gradboostd)\n",
    "print('Accuracy of Gradient Boosting Classifier using Dictionary learned features is...',acc_gb_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of AdaBoost Classifier using HOG features is... 0.998055555556\n",
      "Accuracy of AdaBoost Classifier using Dictionary learned features is... 0.956944444444\n"
     ]
    }
   ],
   "source": [
    "#AdaBoost Classifier\n",
    "clfab = AdaBoostClassifier()\n",
    "clfab.fit(X_train_final, y_train)\n",
    "y_test_predict_adboost=clfab.predict(X_test_final)\n",
    "acc_ab = accuracy_score(y_test, y_test_predict_adboost)\n",
    "print('Accuracy of AdaBoost Classifier using HOG features is...',acc_ab)\n",
    "clfab_d = AdaBoostClassifier()\n",
    "clfab_d.fit(X_train_dict, y_train)\n",
    "y_test_predict_adboostd=clfab_d.predict(X_test_dict)\n",
    "acc_ab_d = accuracy_score(y_test, y_test_predict_adboostd)\n",
    "print('Accuracy of AdaBoost Classifier using Dictionary learned features is...',acc_ab_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Voting Classifier using HOG features is... 0.997166666667\n",
      "Accuracy of Voting Classifier using Dictionary learned features is... 0.998833333333\n"
     ]
    }
   ],
   "source": [
    "#Voting Classifier\n",
    "clfrf = RandomForestClassifier(random_state=1)\n",
    "clfvote = VotingClassifier(estimators=[('rf', clfrf)], voting='hard')\n",
    "clfvote.fit(X_train_final, y_train)\n",
    "y_test_predict_vote=clfvote.predict(X_test_final)\n",
    "acc_v = accuracy_score(y_test, y_test_predict_vote)\n",
    "print('Accuracy of Voting Classifier using HOG features is...',acc_v)\n",
    "clfrfd = RandomForestClassifier(random_state=1)\n",
    "clfvoted = VotingClassifier(estimators=[('rf', clfrfd)], voting='hard')\n",
    "clfvoted.fit(X_train_dict, y_train)\n",
    "y_test_predict_voted=clfvoted.predict(X_test_dict)\n",
    "acc_vd = accuracy_score(y_test, y_test_predict_voted)\n",
    "print('Accuracy of Voting Classifier using Dictionary learned features is...',acc_vd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merging original training and validation data for HOG features...\n",
      "Merging original training and validation data for Dictionary learned features...\n"
     ]
    }
   ],
   "source": [
    "#KFold Cross validation\n",
    "\n",
    "print('Merging original training and validation data for HOG features...')\n",
    "X_full = np.append(X_train_final,X_valid_final,axis=0)\n",
    "\n",
    "print('Merging original training and validation data for Dictionary learned features...')\n",
    "X_full_d = np.append(X_train_dict,X_valid_dict,axis=0)\n",
    "\n",
    "y_full = np.append(y_train,y_valid)\n",
    "\n",
    "kf = KFold(n_splits=2)  #n_splits can be increased, but will result in higher running time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63000, 10)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(X_full_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN: [27000 27001 27002 ..., 53997 53998 53999] TEST: [    0     1     2 ..., 26997 26998 26999]\n",
      "# Tuning hyper-parameters for precision\n",
      "\n",
      "Best parameters set found on development set, using HOG features:\n",
      "\n",
      "{'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using HOG features:\n",
      "\n",
      "0.911 (+/-0.145) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.879 (+/-0.149) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.916 (+/-0.196) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.921 (+/-0.161) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.904 (+/-0.173) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.918 (+/-0.178) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.909 (+/-0.174) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.921 (+/-0.189) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using HOG features.\n",
      "The scores are computed on the full evaluation set, using HOG features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      0.94      0.96     15000\n",
      "          1       0.92      0.99      0.95     12000\n",
      "\n",
      "avg / total       0.96      0.96      0.96     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for recall\n",
      "\n",
      "Best parameters set found on development set, using HOG features:\n",
      "\n",
      "{'max_features': 10, 'n_estimators': 3, 'max_depth': 2}\n",
      "\n",
      "Grid scores on development set, using HOG features:\n",
      "\n",
      "0.889 (+/-0.151) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.909 (+/-0.218) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.865 (+/-0.278) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.870 (+/-0.241) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.883 (+/-0.260) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.884 (+/-0.258) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.894 (+/-0.250) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.892 (+/-0.267) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using HOG features.\n",
      "The scores are computed on the full evaluation set, using HOG features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.87      0.89     15000\n",
      "          1       0.85      0.88      0.86     12000\n",
      "\n",
      "avg / total       0.88      0.88      0.88     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for precision\n",
      "\n",
      "Best parameters set found on development set, using Dictionary learned features:\n",
      "\n",
      "{'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using Dictionary learned features:\n",
      "\n",
      "0.891 (+/-0.155) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.910 (+/-0.178) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.884 (+/-0.141) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.901 (+/-0.175) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.913 (+/-0.180) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.917 (+/-0.186) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.919 (+/-0.189) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.909 (+/-0.177) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using Dictionary learned features.\n",
      "The scores are computed on the full evaluation set, using Dictionary learned features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      0.82      0.88     15000\n",
      "          1       0.81      0.93      0.86     12000\n",
      "\n",
      "avg / total       0.88      0.87      0.87     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for recall\n",
      "\n",
      "Best parameters set found on development set, using Dictionary learned features:\n",
      "\n",
      "{'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using Dictionary learned features:\n",
      "\n",
      "0.849 (+/-0.219) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.864 (+/-0.239) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.845 (+/-0.207) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.854 (+/-0.222) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.886 (+/-0.263) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.887 (+/-0.265) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.894 (+/-0.275) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.891 (+/-0.271) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using Dictionary learned features.\n",
      "The scores are computed on the full evaluation set, using Dictionary learned features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      0.74      0.84     15000\n",
      "          1       0.75      0.96      0.84     12000\n",
      "\n",
      "avg / total       0.86      0.84      0.84     27000\n",
      "\n",
      "\n",
      "TRAIN: [    0     1     2 ..., 26997 26998 26999] TEST: [27000 27001 27002 ..., 53997 53998 53999]\n",
      "# Tuning hyper-parameters for precision\n",
      "\n",
      "Best parameters set found on development set, using HOG features:\n",
      "\n",
      "{'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using HOG features:\n",
      "\n",
      "0.800 (+/-0.143) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.780 (+/-0.327) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.755 (+/-0.409) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.841 (+/-0.180) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.884 (+/-0.161) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.886 (+/-0.179) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.888 (+/-0.169) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.895 (+/-0.166) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using HOG features.\n",
      "The scores are computed on the full evaluation set, using HOG features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.85      0.94      0.89     15000\n",
      "          1       0.91      0.80      0.85     12000\n",
      "\n",
      "avg / total       0.88      0.88      0.87     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for recall\n",
      "\n",
      "Best parameters set found on development set, using HOG features:\n",
      "\n",
      "{'max_features': 30, 'n_estimators': 2, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using HOG features:\n",
      "\n",
      "0.810 (+/-0.225) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.815 (+/-0.236) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.777 (+/-0.279) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.815 (+/-0.205) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.835 (+/-0.230) for {'max_features': 10, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.851 (+/-0.237) for {'max_features': 10, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.868 (+/-0.245) for {'max_features': 30, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.852 (+/-0.250) for {'max_features': 30, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using HOG features.\n",
      "The scores are computed on the full evaluation set, using HOG features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.75      0.94      0.83     15000\n",
      "          1       0.89      0.60      0.72     12000\n",
      "\n",
      "avg / total       0.81      0.79      0.78     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for precision\n",
      "\n",
      "Best parameters set found on development set, using Dictionary learned features:\n",
      "\n",
      "{'max_features': 9, 'n_estimators': 2, 'max_depth': 2}\n",
      "\n",
      "Grid scores on development set, using Dictionary learned features:\n",
      "\n",
      "0.848 (+/-0.122) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.918 (+/-0.222) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.925 (+/-0.199) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.924 (+/-0.197) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.922 (+/-0.195) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.915 (+/-0.184) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.908 (+/-0.179) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.912 (+/-0.183) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using Dictionary learned features.\n",
      "The scores are computed on the full evaluation set, using Dictionary learned features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.45      0.62     15000\n",
      "          1       0.59      1.00      0.74     12000\n",
      "\n",
      "avg / total       0.82      0.69      0.68     27000\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for recall\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set, using Dictionary learned features:\n",
      "\n",
      "{'max_features': 4, 'n_estimators': 2, 'max_depth': 4}\n",
      "\n",
      "Grid scores on development set, using Dictionary learned features:\n",
      "\n",
      "0.850 (+/-0.210) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.878 (+/-0.258) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.896 (+/-0.279) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 2}\n",
      "0.887 (+/-0.267) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 2}\n",
      "0.897 (+/-0.273) for {'max_features': 4, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.873 (+/-0.246) for {'max_features': 4, 'n_estimators': 3, 'max_depth': 4}\n",
      "0.887 (+/-0.266) for {'max_features': 9, 'n_estimators': 2, 'max_depth': 4}\n",
      "0.877 (+/-0.256) for {'max_features': 9, 'n_estimators': 3, 'max_depth': 4}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "The model is trained on the full development set, using Dictionary learned features.\n",
      "The scores are computed on the full evaluation set, using Dictionary learned features.\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      0.62      0.76     15000\n",
      "          1       0.68      0.99      0.80     12000\n",
      "\n",
      "avg / total       0.85      0.79      0.78     27000\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Grid search on K=2 fold cross validation dataset\n",
    "for train_index, test_index in kf.split(X_train_final):\n",
    "    \n",
    "    #Generate new cross-validation dataset\n",
    "    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    #reduced dataset with HOG features\n",
    "    X_train_gs, X_test_gs = X_full[train_index], X_full[test_index]\n",
    "    #reduced dataset with Dictionary learned features\n",
    "    X_train_gs_d, X_test_gs_d = X_full_d[train_index], X_full_d[test_index]    \n",
    "    y_train_gs, y_test_gs = y_full[train_index], y_full[test_index]\n",
    "    \n",
    "    #Get optimal hyper-parameters for this CV dataset, using HOG features\n",
    "    \n",
    "    tuned_parameters_rf = [{\"n_estimators\": [2, 3], \"max_depth\": [2, 4], \"max_features\": [10, 30]}] #add max_features also\n",
    "    tuned_parameters_dt = [{\"max_depth\": [2, 4], \"max_features\": [10, 30]}] \n",
    "    tuned_parameters_et = [{\"max_depth\": [2, 4], \"max_features\": [10, 30]}]\n",
    "    tuned_parameters_gb = [{\"n_estimators\": [2, 3], \"max_depth\": [2, 4], \"max_features\": [10, 30]}]\n",
    "    tuned_parameters_ab = [{\"n_estimators\": [2, 3], }]\n",
    "    tuned_parameters_vc = [{\"estimators\": [2, 3]}]\n",
    "    \n",
    "    clf_considered = RandomForestClassifier()\n",
    "    #clf_considered = tree.DecisionTreeClassifier()\n",
    "    #clf_considered = ExtraTreesClassifier()\n",
    "    #clf_considered = GradientBoostingClassifier()\n",
    "    #clf_considered = AdaBoostClassifier()\n",
    "    #clf_considered = VotingClassifier\n",
    "   \n",
    "   \n",
    "    scores = ['precision', 'recall']\n",
    "    for score in scores:\n",
    "        print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "        print()\n",
    "\n",
    "        clf_gs = GridSearchCV(clf_considered,# this handle can be changed for any other classifier\n",
    "                              tuned_parameters_rf,  scoring='%s_macro' % score)\n",
    "        clf_gs.fit(X_train_gs, np.ravel(y_train_gs))\n",
    "\n",
    "        #HOG features\n",
    "        print(\"Best parameters set found on development set, using HOG features:\")\n",
    "        print()\n",
    "        print(clf_gs.best_params_)\n",
    "        print()\n",
    "        print(\"Grid scores on development set, using HOG features:\")\n",
    "        print()\n",
    "        means = clf_gs.cv_results_['mean_test_score']\n",
    "        stds = clf_gs.cv_results_['std_test_score']\n",
    "        for mean, std, params in zip(means, stds, clf_gs.cv_results_['params']):\n",
    "            print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "                  % (mean, std * 2, params))\n",
    "        print()\n",
    "\n",
    "        print(\"Detailed classification report:\")\n",
    "        print()\n",
    "        print(\"The model is trained on the full development set, using HOG features.\")\n",
    "        print(\"The scores are computed on the full evaluation set, using HOG features.\")\n",
    "        print()\n",
    "        y_true, y_pred = y_test_gs, clf_gs.predict(X_test_gs)\n",
    "        print(classification_report(y_true, y_pred))\n",
    "        print()\n",
    "    \n",
    "        \n",
    "        \n",
    "    #Get optimal hyper-parameters for this CV dataset using Dictionary learned features\n",
    "    tuned_parameters_rf_dl = [{\"n_estimators\": [2, 3], \"max_depth\": [2, 4], \"max_features\": [4, 9]}] #add max_features also\n",
    "    tuned_parameters_dt_dl = [{\"max_depth\": [2, 4], \"max_features\": [4, 9]}] \n",
    "    tuned_parameters_et_dl = [{\"max_depth\": [2, 4], \"max_features\": [4, 9]}]\n",
    "    tuned_parameters_gb_dl = [{\"n_estimators\": [2, 3], \"max_depth\": [2, 4], \"max_features\": [4, 9]}]\n",
    "    tuned_parameters_ab_dl = [{\"n_estimators\": [2, 3]}]\n",
    "    tuned_parameters_vc_dl = [{\"estimators\": [2, 3]}]\n",
    "    scores = ['precision', 'recall']\n",
    "    for score in scores:\n",
    "        print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "        print()\n",
    "\n",
    "      \n",
    "        #Dictionary learned features\n",
    "        clf_gs_d = GridSearchCV(clf_considered,# this handle can be changed for any other classifier\n",
    "                              tuned_parameters_rf_dl,  scoring='%s_macro' % score)\n",
    "        clf_gs_d.fit(X_train_gs_d, y_train_gs)\n",
    "        print(\"Best parameters set found on development set, using Dictionary learned features:\")\n",
    "        print()\n",
    "        print(clf_gs_d.best_params_)\n",
    "        print()\n",
    "        print(\"Grid scores on development set, using Dictionary learned features:\")\n",
    "        print()\n",
    "        means = clf_gs_d.cv_results_['mean_test_score']\n",
    "        stds = clf_gs_d.cv_results_['std_test_score']\n",
    "        for mean, std, params in zip(means, stds, clf_gs_d.cv_results_['params']):\n",
    "            print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "                  % (mean, std * 2, params))\n",
    "        print()\n",
    "\n",
    "        print(\"Detailed classification report:\")\n",
    "        print()\n",
    "        print(\"The model is trained on the full development set, using Dictionary learned features.\")\n",
    "        print(\"The scores are computed on the full evaluation set, using Dictionary learned features.\")\n",
    "        print()\n",
    "        y_true_d, y_pred_d = y_test_gs, clf_gs_d.predict(X_test_gs_d)\n",
    "        print(classification_report(y_true_d, y_pred_d))\n",
    "        print()\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}