1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import ensemble, preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn import ensemble, preprocessing, grid_search, cross_validation\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124.ipynb               imp.txt                 test.csv\r\n",
      "\u001b[34mBlend_op\u001b[m\u001b[m/               imp_feat.txt            test5fold_lol.csv\r\n",
      "\u001b[34mLatest best\u001b[m\u001b[m/            sample.csv              train.csv\r\n",
      "Mergeop.ipynb           sample1.csv             train1.csv\r\n",
      "Untitled.ipynb          sample123.csv           train5fold_lol.csv\r\n",
      "Untitled1.ipynb         sample2.csv             xgb.fmap\r\n",
      "Untitled2.ipynb         sample3.csv             xgbmodel.txt\r\n",
      "blend.csv               sample_submissions.csv\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')#,nrows = 100)\n",
    "test = pd.read_csv('test.csv')#,nrows =100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['target']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i for i in train.columns if i not in test.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(348978, 51)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transaction_id</th>\n",
       "      <th>num_var_1</th>\n",
       "      <th>num_var_2</th>\n",
       "      <th>num_var_3</th>\n",
       "      <th>num_var_4</th>\n",
       "      <th>num_var_5</th>\n",
       "      <th>num_var_6</th>\n",
       "      <th>num_var_7</th>\n",
       "      <th>cat_var_1</th>\n",
       "      <th>cat_var_2</th>\n",
       "      <th>...</th>\n",
       "      <th>cat_var_34</th>\n",
       "      <th>cat_var_35</th>\n",
       "      <th>cat_var_36</th>\n",
       "      <th>cat_var_37</th>\n",
       "      <th>cat_var_38</th>\n",
       "      <th>cat_var_39</th>\n",
       "      <th>cat_var_40</th>\n",
       "      <th>cat_var_41</th>\n",
       "      <th>cat_var_42</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_11</td>\n",
       "      <td>2.302632e-08</td>\n",
       "      <td>0.040182</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.800000e-07</td>\n",
       "      <td>2.302632e-08</td>\n",
       "      <td>2.368421e-08</td>\n",
       "      <td>1.115205e-08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_33</td>\n",
       "      <td>7.965789e-06</td>\n",
       "      <td>0.157872</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.105000e-06</td>\n",
       "      <td>2.769737e-07</td>\n",
       "      <td>7.965789e-06</td>\n",
       "      <td>2.433058e-06</td>\n",
       "      <td>da</td>\n",
       "      <td>tn</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_51</td>\n",
       "      <td>7.828947e-08</td>\n",
       "      <td>0.089140</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.550000e-07</td>\n",
       "      <td>4.671053e-08</td>\n",
       "      <td>1.052632e-07</td>\n",
       "      <td>4.276014e-07</td>\n",
       "      <td>gf</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_54</td>\n",
       "      <td>7.894737e-08</td>\n",
       "      <td>0.227239</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.050000e-06</td>\n",
       "      <td>1.381579e-07</td>\n",
       "      <td>2.190789e-07</td>\n",
       "      <td>1.848054e-08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_62</td>\n",
       "      <td>3.321053e-06</td>\n",
       "      <td>0.160410</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.105000e-06</td>\n",
       "      <td>2.769737e-07</td>\n",
       "      <td>3.340789e-06</td>\n",
       "      <td>2.152983e-06</td>\n",
       "      <td>da</td>\n",
       "      <td>tn</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>id_67</td>\n",
       "      <td>1.953947e-07</td>\n",
       "      <td>0.078566</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.550000e-07</td>\n",
       "      <td>4.671053e-08</td>\n",
       "      <td>4.407895e-08</td>\n",
       "      <td>9.463310e-08</td>\n",
       "      <td>gf</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>id_71</td>\n",
       "      <td>1.736842e-07</td>\n",
       "      <td>0.600508</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.575000e-06</td>\n",
       "      <td>5.986842e-07</td>\n",
       "      <td>1.736842e-07</td>\n",
       "      <td>8.411831e-08</td>\n",
       "      <td>hn</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>id_88</td>\n",
       "      <td>5.789474e-08</td>\n",
       "      <td>0.152374</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.105000e-06</td>\n",
       "      <td>2.769737e-07</td>\n",
       "      <td>5.789474e-08</td>\n",
       "      <td>2.453451e-08</td>\n",
       "      <td>da</td>\n",
       "      <td>tn</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>id_95</td>\n",
       "      <td>3.289474e-08</td>\n",
       "      <td>0.093053</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.500000e-07</td>\n",
       "      <td>8.223684e-08</td>\n",
       "      <td>9.868421e-08</td>\n",
       "      <td>1.274520e-08</td>\n",
       "      <td>pu</td>\n",
       "      <td>ce</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>id_97</td>\n",
       "      <td>5.500000e-07</td>\n",
       "      <td>0.158401</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.105000e-06</td>\n",
       "      <td>2.769737e-07</td>\n",
       "      <td>5.500000e-07</td>\n",
       "      <td>3.138505e-07</td>\n",
       "      <td>da</td>\n",
       "      <td>tn</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  transaction_id     num_var_1  num_var_2  num_var_3     num_var_4  \\\n",
       "0          id_11  2.302632e-08   0.040182        0.0  1.800000e-07   \n",
       "1          id_33  7.965789e-06   0.157872        0.0  2.105000e-06   \n",
       "2          id_51  7.828947e-08   0.089140        0.0  3.550000e-07   \n",
       "3          id_54  7.894737e-08   0.227239        0.0  1.050000e-06   \n",
       "4          id_62  3.321053e-06   0.160410        0.0  2.105000e-06   \n",
       "5          id_67  1.953947e-07   0.078566        0.0  3.550000e-07   \n",
       "6          id_71  1.736842e-07   0.600508        0.0  4.575000e-06   \n",
       "7          id_88  5.789474e-08   0.152374        0.0  2.105000e-06   \n",
       "8          id_95  3.289474e-08   0.093053        0.0  7.500000e-07   \n",
       "9          id_97  5.500000e-07   0.158401        0.0  2.105000e-06   \n",
       "\n",
       "      num_var_5     num_var_6     num_var_7 cat_var_1 cat_var_2  ...    \\\n",
       "0  2.302632e-08  2.368421e-08  1.115205e-08       NaN        ce  ...     \n",
       "1  2.769737e-07  7.965789e-06  2.433058e-06        da        tn  ...     \n",
       "2  4.671053e-08  1.052632e-07  4.276014e-07        gf        ce  ...     \n",
       "3  1.381579e-07  2.190789e-07  1.848054e-08       NaN        ce  ...     \n",
       "4  2.769737e-07  3.340789e-06  2.152983e-06        da        tn  ...     \n",
       "5  4.671053e-08  4.407895e-08  9.463310e-08        gf        ce  ...     \n",
       "6  5.986842e-07  1.736842e-07  8.411831e-08        hn        ce  ...     \n",
       "7  2.769737e-07  5.789474e-08  2.453451e-08        da        tn  ...     \n",
       "8  8.223684e-08  9.868421e-08  1.274520e-08        pu        ce  ...     \n",
       "9  2.769737e-07  5.500000e-07  3.138505e-07        da        tn  ...     \n",
       "\n",
       "  cat_var_34 cat_var_35 cat_var_36 cat_var_37 cat_var_38 cat_var_39  \\\n",
       "0          0          0          0          0          0          0   \n",
       "1          0          0          0          0          0          0   \n",
       "2          0          0          0          0          0          0   \n",
       "3          0          0          0          0          0          0   \n",
       "4          0          0          0          0          0          0   \n",
       "5          0          0          0          0          0          0   \n",
       "6          0          0          0          0          0          0   \n",
       "7          0          0          0          0          0          0   \n",
       "8          0          0          0          0          0          0   \n",
       "9          0          0          0          0          0          0   \n",
       "\n",
       "  cat_var_40 cat_var_41 cat_var_42 target  \n",
       "0          0          0          0      0  \n",
       "1          0          0          0      0  \n",
       "2          0          0          0      0  \n",
       "3          0          0          0      0  \n",
       "4          0          0          0      0  \n",
       "5          0          0          0      0  \n",
       "6          0          0          0      0  \n",
       "7          0          0          0      0  \n",
       "8          0          0          0      0  \n",
       "9          0          0          0      0  \n",
       "\n",
       "[10 rows x 51 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.3389531149646761"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.target.value_counts()[0]*1.0/train.target.value_counts()[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for i in train.columns:\n",
    "#   if train[i].dtype != 'object':\n",
    "#        continue\n",
    "#    print len(np.unique(np.union1d(np.unique(test[i]),np.unique(train[i])))), len(np.setdiff1d(np.unique(test[i]),np.unique(train[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_more = []\n",
    "col_less = []\n",
    "col_num = []\n",
    "train.replace(np.nan,'ksj',inplace= True)\n",
    "test.replace(np.nan,'ksj',inplace= True)\n",
    "for i in train.columns:\n",
    "    if train[i].dtype == 'object':\n",
    "        if len(np.unique(list(train[i])+list(test[i]))) >30:\n",
    "            col_more.append(i)\n",
    "        else:\n",
    "            col_less.append(i)\n",
    "        #print i, len(np.unique(list(train[i])+list(test[i]))),len(np.setdiff1d(np.unique(test[i]),np.unique(train[i])))\n",
    "    else:\n",
    "        col_num.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_more.remove('transaction_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def consolidate(df, col, threshold = 20):\n",
    "    raw = df[col].copy()\n",
    "    cnt = raw.value_counts()\n",
    "    low_cardinal = list(cnt[cnt < threshold].index)\n",
    "    raw[raw.isin(low_cardinal)] = \"OTHER\"\n",
    "    return np.array(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nc = ['num_var_1','num_var_2','num_var_3','num_var_4','num_var_5','num_var_6','num_var_7']\\nfor i in c:    \\n    print i\\n    df = pd.concat((train[[i]],test[[i]]))\\n    vals = df[i].copy().dropna().sort_values().round(4)\\n    vals = pd.rolling_apply(vals, 2, lambda x: x[1] - x[0])\\n    vals = vals[vals > 1e-3]\\n    denom = vals.value_counts().idxmax() \\n    train[i] = np.round(np.array(train[i]/denom),0).astype(int)\\n    test[i] = np.round(np.array(test[i]/denom),0).astype(int)\\n\""
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "c = ['num_var_1','num_var_2','num_var_3','num_var_4','num_var_5','num_var_6','num_var_7']\n",
    "for i in c:    \n",
    "    print i\n",
    "    df = pd.concat((train[[i]],test[[i]]))\n",
    "    vals = df[i].copy().dropna().sort_values().round(4)\n",
    "    vals = pd.rolling_apply(vals, 2, lambda x: x[1] - x[0])\n",
    "    vals = vals[vals > 1e-3]\n",
    "    denom = vals.value_counts().idxmax() \n",
    "    train[i] = np.round(np.array(train[i]/denom),0).astype(int)\n",
    "    test[i] = np.round(np.array(test[i]/denom),0).astype(int)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_var_1 cat_var_2\n",
      "cat_var_1 cat_var_3\n",
      "cat_var_1 cat_var_6\n",
      "cat_var_1 cat_var_8\n",
      "cat_var_1 cat_var_13\n",
      "cat_var_2 cat_var_3\n",
      "cat_var_2 cat_var_6\n",
      "cat_var_2 cat_var_8\n",
      "cat_var_2 cat_var_13\n",
      "cat_var_3 cat_var_6\n",
      "cat_var_3 cat_var_8\n",
      "cat_var_3 cat_var_13\n",
      "cat_var_6 cat_var_8\n",
      "cat_var_6 cat_var_13\n",
      "cat_var_8 cat_var_13\n"
     ]
    }
   ],
   "source": [
    "for enu,i in enumerate(col_more):\n",
    "    train[i] = consolidate(train,i)\n",
    "    test[i] = consolidate(test,i)\n",
    "    for j in col_more[enu+1:]:\n",
    "        print i,j\n",
    "        total = pd.concat((train[['transaction_id',i,j]],test[['transaction_id',i,j]]))\n",
    "        k = total.groupby([i,j]).agg('count').reset_index()\n",
    "        k.columns = [i,j,i+'_'+j+'_count'] \n",
    "        k1 = k.groupby([i]).agg('count')[j].reset_index()\n",
    "        k1.columns = [i,i+'_'+j+'_count1']\n",
    "        k2 = k.groupby([j]).agg('count')[i].reset_index()\n",
    "        k2.columns = [j,j+'_'+i+'_count1']\n",
    "        train = train.merge(k,on=[i,j],how='left')\n",
    "        test = test.merge(k,on=[i,j],how='left')\n",
    "        train = train.merge(k1,on=[i],how='left')\n",
    "        test = test.merge(k1,on=[i],how='left')\n",
    "        train = train.merge(k2,on=[j],how='left')\n",
    "        test = test.merge(k2,on=[j],how='left')\n",
    "    \n",
    "    \n",
    "    \n",
    "        \n",
    "#train = train.drop(col_more,axis=1)\n",
    "#test = test.drop(col_more,axis=1)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ncol_num.remove('target')\\nfrom itertools import combinations\\nfor p,q in list(combinations(range(len(col_num)),2)):\\n    print p,q\\n    i = col_num[p]\\n    j = col_num[q]\\n    train[str(i)+str(j)] = train[i]*train[j]\\n    test[str(i)+str(j)] = test[i]*test[j]\\n        \\n\""
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "col_num.remove('target')\n",
    "from itertools import combinations\n",
    "for p,q in list(combinations(range(len(col_num)),2)):\n",
    "    print p,q\n",
    "    i = col_num[p]\n",
    "    j = col_num[q]\n",
    "    train[str(i)+str(j)] = train[i]*train[j]\n",
    "    test[str(i)+str(j)] = test[i]*test[j]\n",
    "        \n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = train.target.values\n",
    "id_test = test.transaction_id.values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "enc = OneHotEncoder()\n",
    "text_columns = []\n",
    "for f in train.columns:\n",
    "    if train[f].dtype== 'object': \n",
    "        text_columns.append(f)            \n",
    "        lbl = preprocessing.LabelEncoder()\n",
    "        lbl.fit(list(train[f].values) + list(test[f].values))\n",
    "        train[f] = lbl.transform(list(train[f].values))\n",
    "        test[f] = lbl.transform(list(test[f].values)) \n",
    "train = train.drop(['transaction_id','target'],axis =1)\n",
    "test = test.drop(['transaction_id'],axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "def ceate_feature_map(features):\n",
    "    outfile = open('xgb.fmap', 'w')\n",
    "    i = 0\n",
    "    for feat in features:\n",
    "        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n",
    "        i = i + 1\n",
    "\n",
    "    outfile.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import  roc_auc_score,roc_auc_score\n",
    "def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0,max_depth=10):\n",
    "        params = {}\n",
    "        params[\"objective\"] = \"binary:logistic\"\n",
    "        params['eval_metric'] = 'auc'\n",
    "        params[\"eta\"] = 0.01 #0.00334\n",
    "        params[\"min_child_weight\"] = 1\n",
    "        params[\"subsample\"] = 0.8\n",
    "        params[\"colsample_bytree\"] = 0.3\n",
    "        params[\"silent\"] = 1\n",
    "        params[\"max_depth\"] = max_depth\n",
    "        params[\"seed\"] = seed_val\n",
    "        #params[\"max_delta_step\"] = 2\n",
    "        params[\"verbose\"] = 100\n",
    "        num_rounds = 5000 #2500\n",
    "\n",
    "        plst = list(params.items())\n",
    "        xgtrain = xgb.DMatrix(train_X, label=train_y)\n",
    "\n",
    "        if test_y is not None:\n",
    "                xgtest = xgb.DMatrix(test_X, label=test_y)\n",
    "                watchlist = [ (xgtrain,'train'), (xgtest, 'test') ]\n",
    "                model = xgb.train(plst , xgtrain, num_rounds, watchlist, early_stopping_rounds= 500)\n",
    "        else:\n",
    "                xgtest = xgb.DMatrix(test_X)\n",
    "                #xgtest1 = xgb.DMatrix(test_X1)\n",
    "                model = xgb.train(plst, xgtrain, 2000)\n",
    "        #print feature_names,if feature_names\n",
    "        if feature_names:\n",
    "                        create_feature_map(feature_names)\n",
    "                        model.dump_model('xgbmodel.txt', 'xgb.fmap', with_stats=True)\n",
    "                        importance = model.get_fscore(fmap='xgb.fmap')\n",
    "                        importance = sorted(importance.items(), key=operator.itemgetter(1), reverse=True)\n",
    "                        imp_df = pd.DataFrame(importance, columns=['feature','fscore'])\n",
    "                        imp_df['fscore'] = imp_df['fscore'] / imp_df['fscore'].sum()\n",
    "                        imp_df.to_csv(\"imp_feat.txt\", index=False)\n",
    "\n",
    "        pred_test_y = model.predict(xgtest)\n",
    "        \n",
    "        if test_y is not None:\n",
    "                loss = roc_auc_score(test_y, pred_test_y)\n",
    "                print loss\n",
    "        \treturn pred_test_y, loss\n",
    "\telse:\n",
    "\t\treturn pred_test_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n#0.729290022223\\n#0.729594073648\\nfrom sklearn.model_selection import train_test_split\\n\\nX_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.3, random_state=42)\\n\\n#xgb_test = np.array(test_final)\\n#0.603178481377\\nk = runXGB(X_train, y_train, X_test,y_test)\\n'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "#0.729290022223\n",
    "#0.729594073648\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.3, random_state=42)\n",
    "\n",
    "#xgb_test = np.array(test_final)\n",
    "#0.603178481377\n",
    "k = runXGB(X_train, y_train, X_test,y_test)\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yes\n"
     ]
    }
   ],
   "source": [
    "pred1 = runXGB(train, y, test,max_depth=10)\n",
    "pd.DataFrame({'transaction_id':id_test,'target':pred1})[['transaction_id','target']].to_csv('sample1.csv',index =False)\n",
    "\n",
    "print ('yes')\n",
    "pred2 = runXGB(train, y, test,max_depth=12)\n",
    "pd.DataFrame({'transaction_id':id_test,'target':pred2})[['transaction_id','target']].to_csv('sample2.csv',index =False)\n",
    "\n",
    "pred3 = runXGB(train, y, test,max_depth=6)\n",
    "pd.DataFrame({'transaction_id':id_test,'target':pred2})[['transaction_id','target']].to_csv('sample3.csv',index =False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = (pred1+pred2+pred3)*1.0/3\n",
    "pd.DataFrame({'transaction_id':id_test,'target':pred})[['transaction_id','target']].to_csv('sample.csv',index =False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "denormalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}