KNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.metrics import accuracy_score,roc_auc_score\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "#from analytics_tools.analytics_tools3 import freq_discrete\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('Data/churn.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V_ACCT_LGHT</th>\n",
       "      <th>C_IP</th>\n",
       "      <th>C_VMP</th>\n",
       "      <th>V_N_VM_MESS</th>\n",
       "      <th>V_T_DAY_CHG</th>\n",
       "      <th>V_T_EVE_CALLS</th>\n",
       "      <th>V_T_EVE_CHG</th>\n",
       "      <th>V_T_NIG_CALLS</th>\n",
       "      <th>V_T_NIG_CHG</th>\n",
       "      <th>V_T_INT_CALLS</th>\n",
       "      <th>V_T_INT_CHG</th>\n",
       "      <th>V_CS_CALLS</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>25</td>\n",
       "      <td>45.07</td>\n",
       "      <td>99</td>\n",
       "      <td>16.78</td>\n",
       "      <td>91</td>\n",
       "      <td>11.01</td>\n",
       "      <td>3</td>\n",
       "      <td>2.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>107</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>26</td>\n",
       "      <td>27.47</td>\n",
       "      <td>103</td>\n",
       "      <td>16.62</td>\n",
       "      <td>103</td>\n",
       "      <td>11.45</td>\n",
       "      <td>3</td>\n",
       "      <td>3.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>137</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>41.38</td>\n",
       "      <td>110</td>\n",
       "      <td>10.30</td>\n",
       "      <td>104</td>\n",
       "      <td>7.32</td>\n",
       "      <td>5</td>\n",
       "      <td>3.29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>65</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>21.95</td>\n",
       "      <td>83</td>\n",
       "      <td>19.42</td>\n",
       "      <td>111</td>\n",
       "      <td>9.40</td>\n",
       "      <td>6</td>\n",
       "      <td>3.43</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>168</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>21.90</td>\n",
       "      <td>71</td>\n",
       "      <td>8.92</td>\n",
       "      <td>128</td>\n",
       "      <td>6.35</td>\n",
       "      <td>2</td>\n",
       "      <td>3.02</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   V_ACCT_LGHT C_IP C_VMP  V_N_VM_MESS  V_T_DAY_CHG  V_T_EVE_CALLS  \\\n",
       "0          128   No   Yes           25        45.07             99   \n",
       "1          107   No   Yes           26        27.47            103   \n",
       "2          137   No    No            0        41.38            110   \n",
       "3           65   No    No            0        21.95             83   \n",
       "4          168   No    No            0        21.90             71   \n",
       "\n",
       "   V_T_EVE_CHG  V_T_NIG_CALLS  V_T_NIG_CHG  V_T_INT_CALLS  V_T_INT_CHG  \\\n",
       "0        16.78             91        11.01              3         2.70   \n",
       "1        16.62            103        11.45              3         3.70   \n",
       "2        10.30            104         7.32              5         3.29   \n",
       "3        19.42            111         9.40              6         3.43   \n",
       "4         8.92            128         6.35              2         3.02   \n",
       "\n",
       "   V_CS_CALLS  TARGET  ID  \n",
       "0           1       0   1  \n",
       "1           1       0   2  \n",
       "2           0       0   3  \n",
       "3           4       1  11  \n",
       "4           1       0  13  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "var_disc = [v for v in df.columns if v[:2]=='C_']\n",
    "var_cont = [v for v in df.columns if v[:2]=='V_']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for v in var_disc:\n",
    "    df[v] = (df[v]=='Yes').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V_ACCT_LGHT</th>\n",
       "      <th>C_IP</th>\n",
       "      <th>C_VMP</th>\n",
       "      <th>V_N_VM_MESS</th>\n",
       "      <th>V_T_DAY_CHG</th>\n",
       "      <th>V_T_EVE_CALLS</th>\n",
       "      <th>V_T_EVE_CHG</th>\n",
       "      <th>V_T_NIG_CALLS</th>\n",
       "      <th>V_T_NIG_CHG</th>\n",
       "      <th>V_T_INT_CALLS</th>\n",
       "      <th>V_T_INT_CHG</th>\n",
       "      <th>V_CS_CALLS</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>45.07</td>\n",
       "      <td>99</td>\n",
       "      <td>16.78</td>\n",
       "      <td>91</td>\n",
       "      <td>11.01</td>\n",
       "      <td>3</td>\n",
       "      <td>2.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>27.47</td>\n",
       "      <td>103</td>\n",
       "      <td>16.62</td>\n",
       "      <td>103</td>\n",
       "      <td>11.45</td>\n",
       "      <td>3</td>\n",
       "      <td>3.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>137</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>41.38</td>\n",
       "      <td>110</td>\n",
       "      <td>10.30</td>\n",
       "      <td>104</td>\n",
       "      <td>7.32</td>\n",
       "      <td>5</td>\n",
       "      <td>3.29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>65</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.95</td>\n",
       "      <td>83</td>\n",
       "      <td>19.42</td>\n",
       "      <td>111</td>\n",
       "      <td>9.40</td>\n",
       "      <td>6</td>\n",
       "      <td>3.43</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>168</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.90</td>\n",
       "      <td>71</td>\n",
       "      <td>8.92</td>\n",
       "      <td>128</td>\n",
       "      <td>6.35</td>\n",
       "      <td>2</td>\n",
       "      <td>3.02</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   V_ACCT_LGHT  C_IP  C_VMP  V_N_VM_MESS  V_T_DAY_CHG  V_T_EVE_CALLS  \\\n",
       "0          128     0      1           25        45.07             99   \n",
       "1          107     0      1           26        27.47            103   \n",
       "2          137     0      0            0        41.38            110   \n",
       "3           65     0      0            0        21.95             83   \n",
       "4          168     0      0            0        21.90             71   \n",
       "\n",
       "   V_T_EVE_CHG  V_T_NIG_CALLS  V_T_NIG_CHG  V_T_INT_CALLS  V_T_INT_CHG  \\\n",
       "0        16.78             91        11.01              3         2.70   \n",
       "1        16.62            103        11.45              3         3.70   \n",
       "2        10.30            104         7.32              5         3.29   \n",
       "3        19.42            111         9.40              6         3.43   \n",
       "4         8.92            128         6.35              2         3.02   \n",
       "\n",
       "   V_CS_CALLS  TARGET  ID  \n",
       "0           1       0   1  \n",
       "1           1       0   2  \n",
       "2           0       0   3  \n",
       "3           4       1  11  \n",
       "4           1       0  13  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[var_cont+var_disc].copy()\n",
    "y = df['TARGET'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X)\n",
    "Xs = pd.DataFrame(scaler.transform(X),columns=X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V_ACCT_LGHT</th>\n",
       "      <th>V_N_VM_MESS</th>\n",
       "      <th>V_T_DAY_CHG</th>\n",
       "      <th>V_T_EVE_CALLS</th>\n",
       "      <th>V_T_EVE_CHG</th>\n",
       "      <th>V_T_NIG_CALLS</th>\n",
       "      <th>V_T_NIG_CHG</th>\n",
       "      <th>V_T_INT_CALLS</th>\n",
       "      <th>V_T_INT_CHG</th>\n",
       "      <th>V_CS_CALLS</th>\n",
       "      <th>C_IP</th>\n",
       "      <th>C_VMP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633880</td>\n",
       "      <td>0.581395</td>\n",
       "      <td>0.841885</td>\n",
       "      <td>0.489362</td>\n",
       "      <td>0.492640</td>\n",
       "      <td>0.377778</td>\n",
       "      <td>0.693383</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.519126</td>\n",
       "      <td>0.604651</td>\n",
       "      <td>0.433248</td>\n",
       "      <td>0.531915</td>\n",
       "      <td>0.484789</td>\n",
       "      <td>0.511111</td>\n",
       "      <td>0.734390</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.776243</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.683060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.756211</td>\n",
       "      <td>0.606383</td>\n",
       "      <td>0.174681</td>\n",
       "      <td>0.522222</td>\n",
       "      <td>0.349487</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.662983</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.289617</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.305085</td>\n",
       "      <td>0.319149</td>\n",
       "      <td>0.622179</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.543336</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.701657</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.303924</td>\n",
       "      <td>0.191489</td>\n",
       "      <td>0.106968</td>\n",
       "      <td>0.788889</td>\n",
       "      <td>0.259087</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.588398</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   V_ACCT_LGHT  V_N_VM_MESS  V_T_DAY_CHG  V_T_EVE_CALLS  V_T_EVE_CHG  \\\n",
       "0     0.633880     0.581395     0.841885       0.489362     0.492640   \n",
       "1     0.519126     0.604651     0.433248       0.531915     0.484789   \n",
       "2     0.683060     0.000000     0.756211       0.606383     0.174681   \n",
       "3     0.289617     0.000000     0.305085       0.319149     0.622179   \n",
       "4     0.852459     0.000000     0.303924       0.191489     0.106968   \n",
       "\n",
       "   V_T_NIG_CALLS  V_T_NIG_CHG  V_T_INT_CALLS  V_T_INT_CHG  V_CS_CALLS  C_IP  \\\n",
       "0       0.377778     0.693383       0.166667     0.500000    0.166667   0.0   \n",
       "1       0.511111     0.734390       0.166667     0.776243    0.166667   0.0   \n",
       "2       0.522222     0.349487       0.333333     0.662983    0.000000   0.0   \n",
       "3       0.600000     0.543336       0.416667     0.701657    0.666667   0.0   \n",
       "4       0.788889     0.259087       0.083333     0.588398    0.166667   0.0   \n",
       "\n",
       "   C_VMP  \n",
       "0    1.0  \n",
       "1    1.0  \n",
       "2    0.0  \n",
       "3    0.0  \n",
       "4    0.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xt,Xv,yt,yv = train_test_split(Xs,y,train_size=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.71 ms, sys: 205 µs, total: 4.92 ms\n",
      "Wall time: 4.67 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(Xt,yt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metricas(model,Xt,Xv,yt,yv):\n",
    "    print \"Accuracy_Train %.2f, Accuracy_Validate %.2f \"%(accuracy_score(y_pred=model.predict(Xt),y_true=yt),accuracy_score(y_pred=model.predict(Xv),y_true=yv))\n",
    "    print \"ROC_Train %.2f, ROC_Validate %.2f \"%(roc_auc_score(y_score=model.predict_proba(Xt)[:,1],y_true=yt),roc_auc_score(y_score=model.predict_proba(Xv)[:,1],y_true=yv))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy_Train 0.93, Accuracy_Validate 0.88 \n",
      "ROC_Train 0.97, ROC_Validate 0.83 \n"
     ]
    }
   ],
   "source": [
    "metricas(model,Xt,Xv,yt,yv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = dict(n_neighbors = range(100,550)\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = RandomizedSearchCV(cv=3,\n",
    "                          verbose=True,\n",
    "                          param_distributions=param_grid,\n",
    "                          estimator=model,\n",
    "                          n_jobs=-1,\n",
    "                          n_iter=100,\n",
    "                          scoring='roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    6.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   31.2s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.62 s, sys: 84.9 ms, total: 1.71 s\n",
      "Wall time: 48 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed:   47.8s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score='raise',\n",
       "          estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "          fit_params={}, iid=True, n_iter=100, n_jobs=-1,\n",
       "          param_distributions={'n_neighbors': [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152...530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549]},\n",
       "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "          return_train_score=True, scoring='roc_auc', verbose=True)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "grid.fit(Xs,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=105, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy_Train 0.87, Accuracy_Validate 0.85 \n",
      "ROC_Train 0.88, ROC_Validate 0.89 \n"
     ]
    }
   ],
   "source": [
    "metricas(grid.best_estimator_,Xt,Xv,yt,yv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([962., 428., 172.,  96.,  62.,  85.,  52.,  45.,  44.,  14.]),\n",
       " array([0.        , 0.04952381, 0.09904762, 0.14857143, 0.19809524,\n",
       "        0.24761905, 0.29714286, 0.34666667, 0.39619048, 0.44571429,\n",
       "        0.4952381 ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADtNJREFUeJzt3X+MpVV9x/H3R7ZotZafE0J3Nx2spIYSG3GqNCamda1RaFgS0ZC0dTXbbNpStcWkbGsTE/2jYBsppsZ2IzZLYvhRasK2Ug3lR4x/QJ1VBIFaRgrubvgxIq5VYu3Gb/+Yg44r7Ny9z8y9zJ73K5nc85zn3HvOyd3MZ5/z/JhUFZKk/rxg2gOQJE2HASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqdWDIAkn0zyRJKvLqs7OcktSR5srye1+iT5aJKFJPckOWfZe7a19g8m2bY205EkjSor3Qmc5PXAd4FrqursVvdh4FtVdXmSncBJVXVZkvOAdwPnAa8Frqqq1yY5GZgH5oAC9gKvrqqnjtT3qaeeWrOzs4MmKEm92bt37zeramaldhtWalBVn08ye1j1VuA3Wnk3cAdwWau/ppZS5c4kJyY5vbW9paq+BZDkFuDNwLVH6nt2dpb5+fmVhihJWibJI6O0G/ccwGlV9WgrPwac1sobgX3L2u1vdc9VL0maksEngdv/9lftiXJJdiSZTzK/uLi4Wh8rSTrMuAHweFvaob0+0eoPAJuXtdvU6p6r/qdU1a6qmququZmZFZewJEljGjcA9gDPXMmzDbhpWf072tVA5wIH21LR54A3JTmpXTH0plYnSZqSFU8CJ7mWpZO4pybZD3wAuBy4Icl24BHg7a35zSxdAbQAPA28C6CqvpXkQ8AXW7sPPnNCWJI0HSteBjpNc3Nz5VVAknR0kuytqrmV2nknsCR1ygCQpE4ZAJLUqRVPAq9nszs/M5V+H778/Kn0K0lHwyMASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KlBAZDkT5Pcl+SrSa5N8qIkZyS5K8lCkuuTHN/avrBtL7T9s6sxAUnSeMYOgCQbgfcAc1V1NnAccDFwBXBlVb0ceArY3t6yHXiq1V/Z2kmSpmToEtAG4GeTbABeDDwKvAG4se3fDVzYylvbNm3/liQZ2L8kaUxjB0BVHQD+BvgGS7/4DwJ7gW9X1aHWbD+wsZU3Avvaew+19qeM278kaZghS0AnsfS/+jOAXwBeArx56ICS7Egyn2R+cXFx6MdJkp7DkCWgNwL/XVWLVfV/wKeB1wEntiUhgE3AgVY+AGwGaPtPAJ48/EOraldVzVXV3MzMzIDhSZKOZEgAfAM4N8mL21r+FuB+4HbgotZmG3BTK+9p27T9t1VVDehfkjTAkHMAd7F0MvdLwL3ts3YBlwGXJllgaY3/6vaWq4FTWv2lwM4B45YkDbRh5SbPrao+AHzgsOqHgNc8S9vvA28b0p8kafV4J7AkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTg0KgCQnJrkxyX8meSDJryc5OcktSR5srye1tkny0SQLSe5Jcs7qTEGSNI6hRwBXAZ+tqlcAvwo8AOwEbq2qM4Fb2zbAW4Az288O4OMD+5YkDTB2ACQ5AXg9cDVAVf2gqr4NbAV2t2a7gQtbeStwTS25Ezgxyeljj1ySNMiQI4AzgEXgH5N8OcknkrwEOK2qHm1tHgNOa+WNwL5l79/f6iRJUzAkADYA5wAfr6pXAd/jx8s9AFRVAXU0H5pkR5L5JPOLi4sDhidJOpIhAbAf2F9Vd7XtG1kKhMefWdppr0+0/QeAzcvev6nV/YSq2lVVc1U1NzMzM2B4kqQjGTsAquoxYF+SX25VW4D7gT3Atla3DbiplfcA72hXA50LHFy2VCRJmrANA9//buBTSY4HHgLexVKo3JBkO/AI8PbW9mbgPGABeLq1lSRNyaAAqKq7gbln2bXlWdoWcMmQ/iRJq8c7gSWpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmd2jDtARyLZnd+Zir9Pnz5+VPpV9L65BGAJHVqcAAkOS7Jl5P8a9s+I8ldSRaSXJ/k+Fb/wra90PbPDu1bkjS+1TgCeC/wwLLtK4Arq+rlwFPA9la/HXiq1V/Z2kmSpmRQACTZBJwPfKJtB3gDcGNrshu4sJW3tm3a/i2tvSRpCoYeAfwt8GfAD9v2KcC3q+pQ294PbGzljcA+gLb/YGsvSZqCsQMgyW8DT1TV3lUcD0l2JJlPMr+4uLiaHy1JWmbIEcDrgAuSPAxcx9LSz1XAiUmeubx0E3CglQ8AmwHa/hOAJw//0KraVVVzVTU3MzMzYHiSpCMZOwCq6s+ralNVzQIXA7dV1e8AtwMXtWbbgJtaeU/bpu2/rapq3P4lScOsxX0AlwGXJllgaY3/6lZ/NXBKq78U2LkGfUuSRrQqdwJX1R3AHa38EPCaZ2nzfeBtq9GfJGk47wSWpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KmxAyDJ5iS3J7k/yX1J3tvqT05yS5IH2+tJrT5JPppkIck9Sc5ZrUlIko7ekCOAQ8D7quos4FzgkiRnATuBW6vqTODWtg3wFuDM9rMD+PiAviVJA40dAFX1aFV9qZX/B3gA2AhsBXa3ZruBC1t5K3BNLbkTODHJ6WOPXJI0yKqcA0gyC7wKuAs4raoebbseA05r5Y3AvmVv29/qJElTMDgAkvwc8M/An1TVd5bvq6oC6ig/b0eS+STzi4uLQ4cnSXoOgwIgyc+w9Mv/U1X16Vb9+DNLO+31iVZ/ANi87O2bWt1PqKpdVTVXVXMzMzNDhidJOoIhVwEFuBp4oKo+smzXHmBbK28DblpW/452NdC5wMFlS0WSpAnbMOC9rwN+D7g3yd2t7i+Ay4EbkmwHHgHe3vbdDJwHLABPA+8a0LckaaCxA6CqvgDkOXZveZb2BVwybn+SpNXlncCS1KkhS0B6npnd+Zmp9f3w5edPrW9J4/EIQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE759wC0Kqb1twj8OwTS+DwCkKROGQCS1CmXgKQxueyl9c4jAEnqlAEgSZ0yACSpU54D0Lo2rXV46VjgEYAkdcoAkKROuQQkrTPTXPbyEtRjiwEgaWS9nXM51gPPJSBJ6pQBIEmdMgAkqVMGgCR1ygCQpE5NPACSvDnJ15IsJNk56f4lSUsmehlokuOAjwG/BewHvphkT1XdP8lxSNIojvV7LiZ9BPAaYKGqHqqqHwDXAVsnPAZJEpMPgI3AvmXb+1udJGnCnnd3AifZAexom99N8rUBH3cq8M3ho1pXnHM/epx3N3POFT+xebTz/sVRGk06AA4Am5dtb2p1P1JVu4Bdq9FZkvmqmluNz1ovnHM/epx3j3OGtZv3pJeAvgicmeSMJMcDFwN7JjwGSRITPgKoqkNJ/hj4HHAc8Mmqum+SY5AkLZn4OYCquhm4eULdrcpS0jrjnPvR47x7nDOs0bxTVWvxuZKk5zkfBSFJnVr3AbDSoyWSvDDJ9W3/XUlmJz/K1TfCvF+f5EtJDiW5aBpjXG0jzPnSJPcnuSfJrUlGuhTu+W6Eef9BknuT3J3kC0nOmsY4V9Ooj4xJ8tYkleSYuDJohO/6nUkW23d9d5LfH9RhVa3bH5ZOJH8deBlwPPAV4KzD2vwR8PetfDFw/bTHPaF5zwKvBK4BLpr2mCc0598EXtzKf9jRd/3zy8oXAJ+d9rjXes6t3UuBzwN3AnPTHveEvut3An+3Wn2u9yOAUR4tsRXY3co3AluSZIJjXAsrzruqHq6qe4AfTmOAa2CUOd9eVU+3zTtZus9kvRtl3t9ZtvkSYL2f2Bv1kTEfAq4Avj/Jwa2hiT8qZ70HwCiPlvhRm6o6BBwETpnI6NZOj4/UONo5bwf+bU1HNBkjzTvJJUm+DnwYeM+ExrZWVpxzknOAzVV1LP2R4lH/jb+1LXPemGTzs+wf2XoPAOmnJPldYA7462mPZVKq6mNV9UvAZcBfTns8aynJC4CPAO+b9lim4F+A2ap6JXALP17dGMt6D4AVHy2xvE2SDcAJwJMTGd3aGWXex5qR5pzkjcD7gQuq6n8nNLa1dLTf9XXAhWs6orW30pxfCpwN3JHkYeBcYM8xcCJ4lEflPLns3/UngFcP6XC9B8Aoj5bYA2xr5YuA26qdTVnHenykxopzTvIq4B9Y+uX/xBTGuBZGmfeZyzbPBx6c4PjWwhHnXFUHq+rUqpqtqlmWzvdcUFXz0xnuqhnluz592eYFwAODepz2me9VOHN+HvBfLJ09f3+r+yBL/yAAXgT8E7AA/AfwsmmPeULz/jWW1hC/x9IRz33THvME5vzvwOPA3e1nz7THPKF5XwXc1+Z8O/Ar0x7zWs/5sLZ3cAxcBTTid/1X7bv+SvuuXzGkP+8ElqROrfclIEnSmAwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI69f+8rngCh0zT1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f80358e20d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(grid.best_estimator_.predict_proba(Xt)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}