XGBoost V5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import mltools as ml\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.genfromtxt('data/X_train.txt', delimiter=None)\n",
    "Y_train = np.genfromtxt('data/Y_train.txt', delimiter=None)\n",
    "X_test = np.genfromtxt('data/X_test.txt', delimiter=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rescaling Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_rescaled, parameters = ml.rescale(X_train)\n",
    "X_test_rescaled, _ = ml.rescale(X_test, parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split data into Train and Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_split_train, X_split_validation, Y_split_train, Y_split_validation = \\\n",
    "                                                train_test_split(X_train_rescaled, Y_train, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Applying XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlf = xgb.XGBRegressor(max_depth=12, \n",
    "                        learning_rate=0.05, \n",
    "                        n_estimators=2000, \n",
    "                        silent=True, \n",
    "                        objective='binary:logistic', \n",
    "                        nthread=6, \n",
    "                        min_child_weight=6, \n",
    "                        max_delta_step=0, \n",
    "                       subssample=0.8,\n",
    "                       colsample_bytree=0.8,\n",
    "                        colsample_bylevel=1,\n",
    "                        scale_pos_weight=1, \n",
    "                        seed=1440, \n",
    "                        eval_metric='auc',\n",
    "                        missing=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-auc:0.692325\n",
      "Will train until validation_0-auc hasn't improved in 50 rounds.\n",
      "[1]\tvalidation_0-auc:0.705501\n",
      "[2]\tvalidation_0-auc:0.711415\n",
      "[3]\tvalidation_0-auc:0.715192\n",
      "[4]\tvalidation_0-auc:0.718691\n",
      "[5]\tvalidation_0-auc:0.730594\n",
      "[6]\tvalidation_0-auc:0.732754\n",
      "[7]\tvalidation_0-auc:0.733368\n",
      "[8]\tvalidation_0-auc:0.73372\n",
      "[9]\tvalidation_0-auc:0.733863\n",
      "[10]\tvalidation_0-auc:0.735174\n",
      "[11]\tvalidation_0-auc:0.736072\n",
      "[12]\tvalidation_0-auc:0.737182\n",
      "[13]\tvalidation_0-auc:0.737081\n",
      "[14]\tvalidation_0-auc:0.737951\n",
      "[15]\tvalidation_0-auc:0.738844\n",
      "[16]\tvalidation_0-auc:0.742076\n",
      "[17]\tvalidation_0-auc:0.74224\n",
      "[18]\tvalidation_0-auc:0.742517\n",
      "[19]\tvalidation_0-auc:0.742332\n",
      "[20]\tvalidation_0-auc:0.742537\n",
      "[21]\tvalidation_0-auc:0.74266\n",
      "[22]\tvalidation_0-auc:0.742819\n",
      "[23]\tvalidation_0-auc:0.74307\n",
      "[24]\tvalidation_0-auc:0.743893\n",
      "[25]\tvalidation_0-auc:0.744542\n",
      "[26]\tvalidation_0-auc:0.744986\n",
      "[27]\tvalidation_0-auc:0.745342\n",
      "[28]\tvalidation_0-auc:0.745321\n",
      "[29]\tvalidation_0-auc:0.745455\n",
      "[30]\tvalidation_0-auc:0.745741\n",
      "[31]\tvalidation_0-auc:0.746755\n",
      "[32]\tvalidation_0-auc:0.747295\n",
      "[33]\tvalidation_0-auc:0.747639\n",
      "[34]\tvalidation_0-auc:0.747746\n",
      "[35]\tvalidation_0-auc:0.748537\n",
      "[36]\tvalidation_0-auc:0.748922\n",
      "[37]\tvalidation_0-auc:0.749422\n",
      "[38]\tvalidation_0-auc:0.749827\n",
      "[39]\tvalidation_0-auc:0.749934\n",
      "[40]\tvalidation_0-auc:0.750368\n",
      "[41]\tvalidation_0-auc:0.750761\n",
      "[42]\tvalidation_0-auc:0.751394\n",
      "[43]\tvalidation_0-auc:0.751509\n",
      "[44]\tvalidation_0-auc:0.752859\n",
      "[45]\tvalidation_0-auc:0.753139\n",
      "[46]\tvalidation_0-auc:0.75329\n",
      "[47]\tvalidation_0-auc:0.753747\n",
      "[48]\tvalidation_0-auc:0.754156\n",
      "[49]\tvalidation_0-auc:0.754161\n",
      "[50]\tvalidation_0-auc:0.754427\n",
      "[51]\tvalidation_0-auc:0.754635\n",
      "[52]\tvalidation_0-auc:0.754742\n",
      "[53]\tvalidation_0-auc:0.754996\n",
      "[54]\tvalidation_0-auc:0.755756\n",
      "[55]\tvalidation_0-auc:0.755823\n",
      "[56]\tvalidation_0-auc:0.756256\n",
      "[57]\tvalidation_0-auc:0.75676\n",
      "[58]\tvalidation_0-auc:0.757089\n",
      "[59]\tvalidation_0-auc:0.757119\n",
      "[60]\tvalidation_0-auc:0.757289\n",
      "[61]\tvalidation_0-auc:0.757572\n",
      "[62]\tvalidation_0-auc:0.75779\n",
      "[63]\tvalidation_0-auc:0.758132\n",
      "[64]\tvalidation_0-auc:0.758352\n",
      "[65]\tvalidation_0-auc:0.75869\n",
      "[66]\tvalidation_0-auc:0.759095\n",
      "[67]\tvalidation_0-auc:0.75917\n",
      "[68]\tvalidation_0-auc:0.759517\n",
      "[69]\tvalidation_0-auc:0.759638\n",
      "[70]\tvalidation_0-auc:0.759879\n",
      "[71]\tvalidation_0-auc:0.759947\n",
      "[72]\tvalidation_0-auc:0.760163\n",
      "[73]\tvalidation_0-auc:0.760299\n",
      "[74]\tvalidation_0-auc:0.760719\n",
      "[75]\tvalidation_0-auc:0.760795\n",
      "[76]\tvalidation_0-auc:0.760866\n",
      "[77]\tvalidation_0-auc:0.760998\n",
      "[78]\tvalidation_0-auc:0.761262\n",
      "[79]\tvalidation_0-auc:0.76139\n",
      "[80]\tvalidation_0-auc:0.761523\n",
      "[81]\tvalidation_0-auc:0.761962\n",
      "[82]\tvalidation_0-auc:0.762081\n",
      "[83]\tvalidation_0-auc:0.762173\n",
      "[84]\tvalidation_0-auc:0.762296\n",
      "[85]\tvalidation_0-auc:0.762464\n",
      "[86]\tvalidation_0-auc:0.762683\n",
      "[87]\tvalidation_0-auc:0.763058\n",
      "[88]\tvalidation_0-auc:0.763284\n",
      "[89]\tvalidation_0-auc:0.763305\n",
      "[90]\tvalidation_0-auc:0.763368\n",
      "[91]\tvalidation_0-auc:0.764038\n",
      "[92]\tvalidation_0-auc:0.764094\n",
      "[93]\tvalidation_0-auc:0.764514\n",
      "[94]\tvalidation_0-auc:0.764714\n",
      "[95]\tvalidation_0-auc:0.76483\n",
      "[96]\tvalidation_0-auc:0.764908\n",
      "[97]\tvalidation_0-auc:0.765016\n",
      "[98]\tvalidation_0-auc:0.76512\n",
      "[99]\tvalidation_0-auc:0.765281\n",
      "[100]\tvalidation_0-auc:0.765398\n",
      "[101]\tvalidation_0-auc:0.765461\n",
      "[102]\tvalidation_0-auc:0.765685\n",
      "[103]\tvalidation_0-auc:0.765675\n",
      "[104]\tvalidation_0-auc:0.765864\n",
      "[105]\tvalidation_0-auc:0.765908\n",
      "[106]\tvalidation_0-auc:0.765934\n",
      "[107]\tvalidation_0-auc:0.766003\n",
      "[108]\tvalidation_0-auc:0.766201\n",
      "[109]\tvalidation_0-auc:0.766395\n",
      "[110]\tvalidation_0-auc:0.766561\n",
      "[111]\tvalidation_0-auc:0.766895\n",
      "[112]\tvalidation_0-auc:0.767051\n",
      "[113]\tvalidation_0-auc:0.76721\n",
      "[114]\tvalidation_0-auc:0.767362\n",
      "[115]\tvalidation_0-auc:0.76756\n",
      "[116]\tvalidation_0-auc:0.767774\n",
      "[117]\tvalidation_0-auc:0.767921\n",
      "[118]\tvalidation_0-auc:0.767948\n",
      "[119]\tvalidation_0-auc:0.768526\n",
      "[120]\tvalidation_0-auc:0.768778\n",
      "[121]\tvalidation_0-auc:0.768818\n",
      "[122]\tvalidation_0-auc:0.768843\n",
      "[123]\tvalidation_0-auc:0.768947\n",
      "[124]\tvalidation_0-auc:0.769031\n",
      "[125]\tvalidation_0-auc:0.769106\n",
      "[126]\tvalidation_0-auc:0.769199\n",
      "[127]\tvalidation_0-auc:0.769372\n",
      "[128]\tvalidation_0-auc:0.769504\n",
      "[129]\tvalidation_0-auc:0.769601\n",
      "[130]\tvalidation_0-auc:0.770164\n",
      "[131]\tvalidation_0-auc:0.77019\n",
      "[132]\tvalidation_0-auc:0.770531\n",
      "[133]\tvalidation_0-auc:0.770749\n",
      "[134]\tvalidation_0-auc:0.770809\n",
      "[135]\tvalidation_0-auc:0.770813\n",
      "[136]\tvalidation_0-auc:0.77092\n",
      "[137]\tvalidation_0-auc:0.770998\n",
      "[138]\tvalidation_0-auc:0.771284\n",
      "[139]\tvalidation_0-auc:0.771568\n",
      "[140]\tvalidation_0-auc:0.771841\n",
      "[141]\tvalidation_0-auc:0.771851\n",
      "[142]\tvalidation_0-auc:0.771881\n",
      "[143]\tvalidation_0-auc:0.77199\n",
      "[144]\tvalidation_0-auc:0.771987\n",
      "[145]\tvalidation_0-auc:0.77209\n",
      "[146]\tvalidation_0-auc:0.772095\n",
      "[147]\tvalidation_0-auc:0.772111\n",
      "[148]\tvalidation_0-auc:0.772193\n",
      "[149]\tvalidation_0-auc:0.772268\n",
      "[150]\tvalidation_0-auc:0.772384\n",
      "[151]\tvalidation_0-auc:0.772489\n",
      "[152]\tvalidation_0-auc:0.772608\n",
      "[153]\tvalidation_0-auc:0.772713\n",
      "[154]\tvalidation_0-auc:0.772813\n",
      "[155]\tvalidation_0-auc:0.77307\n",
      "[156]\tvalidation_0-auc:0.773031\n",
      "[157]\tvalidation_0-auc:0.773134\n",
      "[158]\tvalidation_0-auc:0.773176\n",
      "[159]\tvalidation_0-auc:0.773496\n",
      "[160]\tvalidation_0-auc:0.773508\n",
      "[161]\tvalidation_0-auc:0.773518\n",
      "[162]\tvalidation_0-auc:0.773651\n",
      "[163]\tvalidation_0-auc:0.773706\n",
      "[164]\tvalidation_0-auc:0.773746\n",
      "[165]\tvalidation_0-auc:0.773738\n",
      "[166]\tvalidation_0-auc:0.773843\n",
      "[167]\tvalidation_0-auc:0.773895\n",
      "[168]\tvalidation_0-auc:0.77419\n",
      "[169]\tvalidation_0-auc:0.774226\n",
      "[170]\tvalidation_0-auc:0.774379\n",
      "[171]\tvalidation_0-auc:0.774572\n",
      "[172]\tvalidation_0-auc:0.774848\n",
      "[173]\tvalidation_0-auc:0.774875\n",
      "[174]\tvalidation_0-auc:0.774863\n",
      "[175]\tvalidation_0-auc:0.774863\n",
      "[176]\tvalidation_0-auc:0.774923\n",
      "[177]\tvalidation_0-auc:0.775038\n",
      "[178]\tvalidation_0-auc:0.775186\n",
      "[179]\tvalidation_0-auc:0.775224\n",
      "[180]\tvalidation_0-auc:0.775544\n",
      "[181]\tvalidation_0-auc:0.775729\n",
      "[182]\tvalidation_0-auc:0.77581\n",
      "[183]\tvalidation_0-auc:0.775812\n",
      "[184]\tvalidation_0-auc:0.775841\n",
      "[185]\tvalidation_0-auc:0.775857\n",
      "[186]\tvalidation_0-auc:0.77586\n",
      "[187]\tvalidation_0-auc:0.775877\n",
      "[188]\tvalidation_0-auc:0.776011\n",
      "[189]\tvalidation_0-auc:0.776142\n",
      "[190]\tvalidation_0-auc:0.77617\n",
      "[191]\tvalidation_0-auc:0.7762\n",
      "[192]\tvalidation_0-auc:0.776207\n",
      "[193]\tvalidation_0-auc:0.776202\n",
      "[194]\tvalidation_0-auc:0.776282\n",
      "[195]\tvalidation_0-auc:0.776404\n",
      "[196]\tvalidation_0-auc:0.776408\n",
      "[197]\tvalidation_0-auc:0.776428\n",
      "[198]\tvalidation_0-auc:0.776434\n",
      "[199]\tvalidation_0-auc:0.776505\n",
      "[200]\tvalidation_0-auc:0.77663\n",
      "[201]\tvalidation_0-auc:0.776634\n",
      "[202]\tvalidation_0-auc:0.776784\n",
      "[203]\tvalidation_0-auc:0.776841\n",
      "[204]\tvalidation_0-auc:0.776972\n",
      "[205]\tvalidation_0-auc:0.777017\n",
      "[206]\tvalidation_0-auc:0.777034\n",
      "[207]\tvalidation_0-auc:0.777069\n",
      "[208]\tvalidation_0-auc:0.77718\n",
      "[209]\tvalidation_0-auc:0.77733\n",
      "[210]\tvalidation_0-auc:0.777477\n",
      "[211]\tvalidation_0-auc:0.777544\n",
      "[212]\tvalidation_0-auc:0.777556\n",
      "[213]\tvalidation_0-auc:0.777639\n",
      "[214]\tvalidation_0-auc:0.777822\n",
      "[215]\tvalidation_0-auc:0.777964\n",
      "[216]\tvalidation_0-auc:0.778001\n",
      "[217]\tvalidation_0-auc:0.778043\n",
      "[218]\tvalidation_0-auc:0.778083\n",
      "[219]\tvalidation_0-auc:0.778077\n",
      "[220]\tvalidation_0-auc:0.778077\n",
      "[221]\tvalidation_0-auc:0.778144\n",
      "[222]\tvalidation_0-auc:0.778142\n",
      "[223]\tvalidation_0-auc:0.778143\n",
      "[224]\tvalidation_0-auc:0.778144\n",
      "[225]\tvalidation_0-auc:0.778401\n",
      "[226]\tvalidation_0-auc:0.778508\n",
      "[227]\tvalidation_0-auc:0.778544\n",
      "[228]\tvalidation_0-auc:0.778541\n",
      "[229]\tvalidation_0-auc:0.778618\n",
      "[230]\tvalidation_0-auc:0.77865\n",
      "[231]\tvalidation_0-auc:0.778657\n",
      "[232]\tvalidation_0-auc:0.778778\n",
      "[233]\tvalidation_0-auc:0.778801\n",
      "[234]\tvalidation_0-auc:0.778827\n",
      "[235]\tvalidation_0-auc:0.778851\n",
      "[236]\tvalidation_0-auc:0.778875\n",
      "[237]\tvalidation_0-auc:0.77905\n",
      "[238]\tvalidation_0-auc:0.779148\n",
      "[239]\tvalidation_0-auc:0.779144\n",
      "[240]\tvalidation_0-auc:0.779129\n",
      "[241]\tvalidation_0-auc:0.779162\n",
      "[242]\tvalidation_0-auc:0.779343\n",
      "[243]\tvalidation_0-auc:0.779332\n",
      "[244]\tvalidation_0-auc:0.779446\n",
      "[245]\tvalidation_0-auc:0.779615\n",
      "[246]\tvalidation_0-auc:0.77968\n",
      "[247]\tvalidation_0-auc:0.779744\n",
      "[248]\tvalidation_0-auc:0.779773\n",
      "[249]\tvalidation_0-auc:0.779807\n",
      "[250]\tvalidation_0-auc:0.779878\n",
      "[251]\tvalidation_0-auc:0.77989\n",
      "[252]\tvalidation_0-auc:0.779863\n",
      "[253]\tvalidation_0-auc:0.779904\n",
      "[254]\tvalidation_0-auc:0.779939\n",
      "[255]\tvalidation_0-auc:0.780046\n",
      "[256]\tvalidation_0-auc:0.780073\n",
      "[257]\tvalidation_0-auc:0.780141\n",
      "[258]\tvalidation_0-auc:0.780178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[259]\tvalidation_0-auc:0.780333\n",
      "[260]\tvalidation_0-auc:0.780385\n",
      "[261]\tvalidation_0-auc:0.780394\n",
      "[262]\tvalidation_0-auc:0.780531\n",
      "[263]\tvalidation_0-auc:0.780606\n",
      "[264]\tvalidation_0-auc:0.780707\n",
      "[265]\tvalidation_0-auc:0.780814\n",
      "[266]\tvalidation_0-auc:0.780871\n",
      "[267]\tvalidation_0-auc:0.780875\n",
      "[268]\tvalidation_0-auc:0.780859\n",
      "[269]\tvalidation_0-auc:0.780871\n",
      "[270]\tvalidation_0-auc:0.780944\n",
      "[271]\tvalidation_0-auc:0.780998\n",
      "[272]\tvalidation_0-auc:0.781074\n",
      "[273]\tvalidation_0-auc:0.78107\n",
      "[274]\tvalidation_0-auc:0.781059\n",
      "[275]\tvalidation_0-auc:0.781141\n",
      "[276]\tvalidation_0-auc:0.781335\n",
      "[277]\tvalidation_0-auc:0.781558\n",
      "[278]\tvalidation_0-auc:0.781577\n",
      "[279]\tvalidation_0-auc:0.781618\n",
      "[280]\tvalidation_0-auc:0.781745\n",
      "[281]\tvalidation_0-auc:0.781762\n",
      "[282]\tvalidation_0-auc:0.781763\n",
      "[283]\tvalidation_0-auc:0.781751\n",
      "[284]\tvalidation_0-auc:0.781746\n",
      "[285]\tvalidation_0-auc:0.781787\n",
      "[286]\tvalidation_0-auc:0.78196\n",
      "[287]\tvalidation_0-auc:0.781985\n",
      "[288]\tvalidation_0-auc:0.782128\n",
      "[289]\tvalidation_0-auc:0.782209\n",
      "[290]\tvalidation_0-auc:0.782224\n",
      "[291]\tvalidation_0-auc:0.782272\n",
      "[292]\tvalidation_0-auc:0.782273\n",
      "[293]\tvalidation_0-auc:0.782314\n",
      "[294]\tvalidation_0-auc:0.78234\n",
      "[295]\tvalidation_0-auc:0.782349\n",
      "[296]\tvalidation_0-auc:0.782456\n",
      "[297]\tvalidation_0-auc:0.782452\n",
      "[298]\tvalidation_0-auc:0.782455\n",
      "[299]\tvalidation_0-auc:0.782523\n",
      "[300]\tvalidation_0-auc:0.782543\n",
      "[301]\tvalidation_0-auc:0.782685\n",
      "[302]\tvalidation_0-auc:0.782688\n",
      "[303]\tvalidation_0-auc:0.782677\n",
      "[304]\tvalidation_0-auc:0.782672\n",
      "[305]\tvalidation_0-auc:0.782671\n",
      "[306]\tvalidation_0-auc:0.782717\n",
      "[307]\tvalidation_0-auc:0.782715\n",
      "[308]\tvalidation_0-auc:0.782786\n",
      "[309]\tvalidation_0-auc:0.782864\n",
      "[310]\tvalidation_0-auc:0.783015\n",
      "[311]\tvalidation_0-auc:0.783182\n",
      "[312]\tvalidation_0-auc:0.783182\n",
      "[313]\tvalidation_0-auc:0.783186\n",
      "[314]\tvalidation_0-auc:0.783302\n",
      "[315]\tvalidation_0-auc:0.7833\n",
      "[316]\tvalidation_0-auc:0.783317\n",
      "[317]\tvalidation_0-auc:0.783316\n",
      "[318]\tvalidation_0-auc:0.783336\n",
      "[319]\tvalidation_0-auc:0.783396\n",
      "[320]\tvalidation_0-auc:0.783426\n",
      "[321]\tvalidation_0-auc:0.783424\n",
      "[322]\tvalidation_0-auc:0.783489\n",
      "[323]\tvalidation_0-auc:0.78357\n",
      "[324]\tvalidation_0-auc:0.783655\n",
      "[325]\tvalidation_0-auc:0.783647\n",
      "[326]\tvalidation_0-auc:0.783643\n",
      "[327]\tvalidation_0-auc:0.783707\n",
      "[328]\tvalidation_0-auc:0.783704\n",
      "[329]\tvalidation_0-auc:0.783704\n",
      "[330]\tvalidation_0-auc:0.783725\n",
      "[331]\tvalidation_0-auc:0.783829\n",
      "[332]\tvalidation_0-auc:0.783903\n",
      "[333]\tvalidation_0-auc:0.78403\n",
      "[334]\tvalidation_0-auc:0.784075\n",
      "[335]\tvalidation_0-auc:0.784071\n",
      "[336]\tvalidation_0-auc:0.784115\n",
      "[337]\tvalidation_0-auc:0.78411\n",
      "[338]\tvalidation_0-auc:0.784121\n",
      "[339]\tvalidation_0-auc:0.784185\n",
      "[340]\tvalidation_0-auc:0.784192\n",
      "[341]\tvalidation_0-auc:0.784195\n",
      "[342]\tvalidation_0-auc:0.784183\n",
      "[343]\tvalidation_0-auc:0.784229\n",
      "[344]\tvalidation_0-auc:0.78424\n",
      "[345]\tvalidation_0-auc:0.784269\n",
      "[346]\tvalidation_0-auc:0.784305\n",
      "[347]\tvalidation_0-auc:0.784368\n",
      "[348]\tvalidation_0-auc:0.784364\n",
      "[349]\tvalidation_0-auc:0.784393\n",
      "[350]\tvalidation_0-auc:0.784387\n",
      "[351]\tvalidation_0-auc:0.784393\n",
      "[352]\tvalidation_0-auc:0.784414\n",
      "[353]\tvalidation_0-auc:0.784454\n",
      "[354]\tvalidation_0-auc:0.78445\n",
      "[355]\tvalidation_0-auc:0.784452\n",
      "[356]\tvalidation_0-auc:0.784514\n",
      "[357]\tvalidation_0-auc:0.784543\n",
      "[358]\tvalidation_0-auc:0.78468\n",
      "[359]\tvalidation_0-auc:0.784696\n",
      "[360]\tvalidation_0-auc:0.784696\n",
      "[361]\tvalidation_0-auc:0.784686\n",
      "[362]\tvalidation_0-auc:0.784711\n",
      "[363]\tvalidation_0-auc:0.784754\n",
      "[364]\tvalidation_0-auc:0.784927\n",
      "[365]\tvalidation_0-auc:0.785044\n",
      "[366]\tvalidation_0-auc:0.785042\n",
      "[367]\tvalidation_0-auc:0.785103\n"
     ]
    }
   ],
   "source": [
    "xlf.fit(X_split_train, Y_split_train, eval_metric='auc', verbose = True, \\\n",
    "        eval_set = [(X_split_validation, Y_split_validation)], early_stopping_rounds=50)\n",
    "\n",
    "# xlf.fit(X_train_rescaled, Y_train, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = xlf.predict(X_test_rescaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.82907212,  0.63176537,  0.31591514, ...,  0.27784756,\n",
       "        0.09122714,  0.12314682], dtype=float32)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions #1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "Yte = np.vstack((np.arange(X_test.shape[0]), predictions)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('Y_submit_XGBoost_v5.txt', Yte, '%d, %.2f', header='ID,Prob1', comments='', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}