test_knn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 762)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "\n",
    "train_df = pd.read_csv(\"./input/train.csv\")\n",
    "author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}\n",
    "train_y = train_df['author'].map(author_mapping_dict)\n",
    "train_Y = train_y\n",
    "\n",
    "\n",
    "with open('feat.pkl','rb') as fin:\n",
    "    f_train_X,f_test_X = pickle.load(fin)\n",
    "print(f_train_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def done\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold,KFold\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression,SGDClassifier\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "def cv_test(k_cnt=3, s_flag = False):\n",
    "    rnd = 42\n",
    "    if s_flag:\n",
    "        kf = StratifiedKFold(n_splits=k_cnt, shuffle=True, random_state=rnd)\n",
    "    else:\n",
    "        kf = KFold(n_splits=k_cnt, shuffle=True, random_state=rnd)\n",
    "    test_pred = None\n",
    "    weighted_test_pred = None\n",
    "    org_train_pred = None\n",
    "    avg_k_score = 0\n",
    "    reverse_score = 0\n",
    "    best_loss = 100\n",
    "    best_single_pred = None\n",
    "    for train_index, test_index in kf.split(f_train_X,train_Y):\n",
    "        X_train, X_test = f_train_X[train_index], f_train_X[test_index]\n",
    "        y_train, y_test = train_Y[train_index], train_Y[test_index]\n",
    "        \n",
    "        m = KNeighborsClassifier()\n",
    "        #m = LogisticRegression(solver='newton-cg',multi_class='multinomial')\n",
    "        #m = SGDClassifier()\n",
    "        m.fit(X_train,y_train)\n",
    "        \n",
    "        # get res\n",
    "        train_pred = m.predict_proba(X_train)\n",
    "        print(train_pred.shape,y_train.shape)\n",
    "        valid_pred = m.predict_proba(X_test)\n",
    "        tmp_train_pred = m.predict(f_train_X)\n",
    "        \n",
    "        # cal score\n",
    "        train_score = log_loss(y_train,train_pred)\n",
    "        valid_score = log_loss(y_test,valid_pred)\n",
    "        print('train log loss',train_score,'valid log loss',valid_score)\n",
    "        avg_k_score += valid_score\n",
    "        rev_valid_score = 1.0/valid_score # use for weighted\n",
    "        reverse_score += rev_valid_score # sum\n",
    "        print('rev',rev_valid_score)\n",
    "        \n",
    "        if test_pred is None:\n",
    "            test_pred = m.predict_proba(f_test_X)\n",
    "            weighted_test_pred = test_pred*rev_valid_score\n",
    "            org_train_pred = tmp_train_pred\n",
    "            best_loss = valid_score\n",
    "            best_single_pred = test_pred\n",
    "        else:\n",
    "            curr_pred = m.predict_proba(f_test_X)\n",
    "            test_pred += curr_pred\n",
    "            weighted_test_pred += curr_pred*rev_valid_score # fix bug here\n",
    "            org_train_pred += tmp_train_pred\n",
    "            # find better single model\n",
    "            if valid_score < best_loss:\n",
    "                print('BETTER')\n",
    "                best_loss = valid_score\n",
    "                best_single_pred = curr_pred\n",
    "\n",
    "    # avg\n",
    "    test_pred = test_pred / k_cnt\n",
    "    test_pred = np.round(test_pred,4)\n",
    "    print(test_pred.shape)\n",
    "    print(test_pred[:5])\n",
    "    org_train_pred = org_train_pred / k_cnt\n",
    "    avg_k_score = avg_k_score/k_cnt\n",
    "\n",
    "\n",
    "    submiss=pd.read_csv(\"./input/sample_submission.csv\")\n",
    "    submiss['EAP']=test_pred[:,0]\n",
    "    submiss['HPL']=test_pred[:,1]\n",
    "    submiss['MWS']=test_pred[:,2]\n",
    "    submiss.to_csv(\"results/knn_res_{}_{}.csv\".format(k_cnt, s_flag),index=False)\n",
    "    print(submiss.head(5))\n",
    "    print('--------------')\n",
    "    print(reverse_score)\n",
    "    # weigthed\n",
    "    submiss=pd.read_csv(\"./input/sample_submission.csv\")\n",
    "    weighted_test_pred = weighted_test_pred / reverse_score\n",
    "    weighted_test_pred = np.round(weighted_test_pred,4)\n",
    "    submiss['EAP']=weighted_test_pred[:,0]\n",
    "    submiss['HPL']=weighted_test_pred[:,1]\n",
    "    submiss['MWS']=weighted_test_pred[:,2]\n",
    "    submiss.to_csv(\"results/weighted_knn_res_{}_{}.csv\".format(k_cnt, s_flag),index=False)\n",
    "    print(submiss.head(5))\n",
    "    print('---------------')\n",
    "    # best single\n",
    "    submiss=pd.read_csv(\"./input/sample_submission.csv\")\n",
    "    weighted_test_pred = np.round(best_single_pred,4)\n",
    "    submiss['EAP']=weighted_test_pred[:,0]\n",
    "    submiss['HPL']=weighted_test_pred[:,1]\n",
    "    submiss['MWS']=weighted_test_pred[:,2]\n",
    "    submiss.to_csv(\"results/single_knn_res_{}_{}.csv\".format(k_cnt, s_flag),index=False)\n",
    "    print(submiss.head(5))\n",
    "    print('---------------')\n",
    "    \n",
    "    # train log loss\n",
    "    print('local average valid loss',avg_k_score)\n",
    "    print('train log loss', log_loss(train_Y,org_train_pred))\n",
    "    \n",
    "print('def done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13052, 3) (13052,)\n",
      "train log loss 0.338273498636 valid log loss 1.75062371429\n",
      "rev 0.57122498218\n",
      "(13053, 3) (13053,)\n"
     ]
    }
   ],
   "source": [
    "cv_test(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv_test(5,True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}