ml_on_text_bow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import json\n",
    "import math\n",
    "\n",
    "import numpy\n",
    "import matplotlib.pyplot as plot\n",
    "import pandas\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.grid_search import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('pos_reviews.json') as data_file:    \n",
    "    data_pos = json.load(data_file)\n",
    "with open('neg_reviews.json') as data_file:    \n",
    "    data_neg = json.load(data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_texts = data_pos.keys()\n",
    "neg_texts = data_neg.keys()\n",
    "all_texts = pos_texts + neg_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target = [1]*1000 + [0]*1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer()\n",
    "matrix_count = count_vect.fit_transform(all_texts).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 39659)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix_count.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>00</th>\n",
       "      <th>000</th>\n",
       "      <th>0009f</th>\n",
       "      <th>007</th>\n",
       "      <th>00s</th>\n",
       "      <th>03</th>\n",
       "      <th>04</th>\n",
       "      <th>05</th>\n",
       "      <th>05425</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>zukovsky</th>\n",
       "      <th>zulu</th>\n",
       "      <th>zundel</th>\n",
       "      <th>zurg</th>\n",
       "      <th>zus</th>\n",
       "      <th>zweibel</th>\n",
       "      <th>zwick</th>\n",
       "      <th>zwigoff</th>\n",
       "      <th>zycie</th>\n",
       "      <th>zzzzzzz</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9 </th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td> 0</td>\n",
       "      <td> 2</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 1</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td>...</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "      <td> 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 39659 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    00  000  0009f  007  00s  03  04  05  05425  10   ...     zukovsky  zulu  \\\n",
       "0    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "1    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "2    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "3    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "4    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "5    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "6    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "7    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "8    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "9    0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "10   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "11   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "12   0    2      0    0    0   0   0   0      0   1   ...            0     0   \n",
       "13   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "14   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "15   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "16   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "17   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "18   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "19   0    0      0    0    0   0   0   0      0   0   ...            0     0   \n",
       "\n",
       "    zundel  zurg  zus  zweibel  zwick  zwigoff  zycie  zzzzzzz  \n",
       "0        0     0    0        0      0        0      0        0  \n",
       "1        0     0    0        0      0        0      0        0  \n",
       "2        0     0    0        0      0        0      0        0  \n",
       "3        0     0    0        0      0        0      0        0  \n",
       "4        0     0    0        0      0        0      0        0  \n",
       "5        0     0    0        0      0        0      0        0  \n",
       "6        0     0    0        0      0        0      0        0  \n",
       "7        0     0    0        0      0        0      0        0  \n",
       "8        0     0    0        0      0        0      0        0  \n",
       "9        0     0    0        0      0        0      0        0  \n",
       "10       0     0    0        0      0        0      0        0  \n",
       "11       0     0    0        0      0        0      0        0  \n",
       "12       0     0    0        0      1        0      0        0  \n",
       "13       0     0    0        0      0        0      0        0  \n",
       "14       0     0    0        0      0        0      0        0  \n",
       "15       0     0    0        0      0        0      0        0  \n",
       "16       0     0    0        0      0        0      0        0  \n",
       "17       0     0    0        0      0        0      0        0  \n",
       "18       0     0    0        0      0        0      0        0  \n",
       "19       0     0    0        0      0        0      0        0  \n",
       "\n",
       "[20 rows x 39659 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = [x[0] for x in sorted(count_vect.vocabulary_.items(), key=lambda x: x[1])]\n",
    "pandas.DataFrame(matrix_count, columns=words).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_valid(matrix_counts, target, test_size=0.3, random_state=0):\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(matrix_counts, target, test_size=0.3, random_state=0)\n",
    "    return (X_train, X_test, Y_train, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_count, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, penalty='l2', random_state=None, tol=0.0001)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression()\n",
    "model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict = model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "precision, recall, _ = metrics.precision_recall_curve(Y_test, predict[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "metrics.precision_recall_curve?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 1)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEACAYAAABI5zaHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAE+ZJREFUeJzt3XuQpXV95/H3h+YiiCMohmRhEhAnBDaCmjBBky070Son\n",
       "1JZUzJaImmxuJTE7bi7WSpKqwOwl2U0qFytFlqIQieWWwdXoOtmlIMnqrIkKEeWiccYwIGZmMFQA\n",
       "hQFmYAa/+8dzhj40M31Onz59Tvev36+qp/o8z/n183zPr7o//evnmqpCktSWo6ZdgCRp/Ax3SWqQ\n",
       "4S5JDTLcJalBhrskNchwl6QGDQz3JO9P8kCSLy3Q5o+T3J3kziSvHG+JkqTFGmbkfj2w6UhvJrkI\n",
       "eFlVbQDeAVw9ptokSSMaGO5V9TfANxdo8kbgA722twInJTl1POVJkkYxjn3upwG7+uZ3A6ePYb2S\n",
       "pBGN64Bq5s17TwNJmqKjx7COPcD6vvnTe8ueJYmBL0kjqKr5A+iBxhHuW4HNwA1JLgS+VVUPHK7h\n",
       "KAW2KMmWqtrSvWYPsLHquX8Qj/z9zADPmzcdd5hl/dMM8MEq9o/vkyxdf1+sdfbFHPtizqgD44Hh\n",
       "nuTPgNcCpyTZBVwJHANQVddU1Y1JLkqyE3gc+NlRClnjtiQU8PzDTCdw+KDev8jpjcAXgC8u5wdJ\n",
       "CHAscHxvOlTzsX3TcXOvX3tuwlufu/xwbQcuv7mK/7Kcn09aLQaGe1VdOkSbzeMpZ036D8CpwBN0\n",
       "fxznT08A++gC+sne1wNVizuukfBF4PiEl9D90Tix7+uJ85YdP2963iKXHezVfKjuQ7U/NW96Es5+\n",
       "Wa9937JntXmc7mytI3z/M69fBVyScDtd8PdPh/6zORb4UBX3LqbvpNUok7qfe5Jyt0wnyWxVbZvs\n",
       "Nvlr4EeAx+gC87F5r/u/7uPZ4bxviGXPLK/i6eHrGk9fJHwv8Cd0fygO/RF8sm/aD7wO+J/Ah1h4\n",
       "F9ZipoPA26o4sPTPMPmfi5XKvpgzanYa7mtEQhY72m9Nwm8Av8rid2ktNH0QeC/dGWKH/nu5poq7\n",
       "JvW51DbDXZqChF8BXsLcfy4X050tdgvd8ZLj533dD/xSFd+eSsFadUbNznGcLSOtWVW8t38+YTtd\n",
       "wJ9Jd7zkCeAR5o6dvA94IuE4urA/gbkD54emm6p4z6Q+g9rkyF2aoIR3Ai9gLvgf73v9BHAe8LvA\n",
       "PTz7rKkZ4Fzg/rW+e22tcbeM1ICEY4GNdLtv+s+g+r/ABrozfjZV8ddTK1IT5W4ZqQFVPAX87fzl\n",
       "Ca+hG73/KXBywtF0/wH0T+uAx6r47GK22bs24YTeOo4H7vO/g9XPcJdWgSoeB0jYS3cqZ+hOXd0L\n",
       "PNr7uh+4IOFdPDf4F5pOpLtWYC/wQuDdCe+v4olJfT6Nn7tlpFUk4Ri6K8T3zR9d90bz19IF/97F\n",
       "TFUc7K3jPwHvBD4O7KT7b2D+fwf9888DfgL41mHa/EMVtyxDN6wp7nOXNBYJbwFmmfuPYP7Uv/yP\n",
       "gAsOs3wd3TGCTzIX9tCdSXRUFXsn82lWP8Nd0oqRcBLwVrqDwoeC/3rgu3pNLqzi81Mqb1Ux3CWt\n",
       "aL3dRk8D24B/BB6m28f/QrrR/c101wEcrOLRKZW54hjuklaFhH9Dd87+o3QXeD0CfA/we3QXen0T\n",
       "+PdVfGJqRa4ghrukVa13SubzgauBt9Dd5O3fHjrYu1YZ7pKakfCvgQ/QHYj9RbpHgn6kikemWtgU\n",
       "GO6SmtLbR//7wL8AXgNcVsX/mW5Vk2e4S2pWwoeBNwM/WcXHpl3PJBnukprVe27w+4HvBR4CPlDF\n",
       "R6Zb1WSMmp1HLUcxkjROvad7/RFwDV24XzjdilY+7y0jaVWo4g7gjoQX0+2H1wIcuUtajc5KeHvv\n",
       "bpk6DMNd0mpzF90tiq8APpNw9ZTrWZE8oCppVeo92OTtdFezvmLa9SwXD6hKWlN6Dzb54rTrWKkM\n",
       "d0lN6J0uqR7DXdJq9hTw/QmPAQ9Ou5iVxHCXtJptB84CTgfWJXwh4fgp17QiGO6SVq0qqoqvV/Et\n",
       "4BXAy4AXT7msFcGzZSQ1I+Fe4Ey62wUfpHvy03l0Dwj5NN3FT/cBHzz00PGVznvLSFrzEo4Cfho4\n",
       "Fbgf+AbwcuD1wNeAk+ke//elKs6bVp2LYbhL0hASNgCfAf4d8FG6wH8xsLOKyQTiIoyand5bRtJa\n",
       "80/AJ+l23ewHnqS74vUC4M4p1jVWHlCVtKZUsbeKtwDfCby4ipPobmlwzHQrGy/DXdKaVMUDVTzR\n",
       "t+gtvTtONsFwlyT4H8C7gQcTfmLaxYyDB1QlCUg4DrgOeB7wW1Vsn3JJwDLeOCzJpiQ7ktyd5PLD\n",
       "vH9KkpuS3JHky0l+ZrFFSNK0VfEk8BG6K17fOOVylmzBcE8yA1wFbALOBS5Ncs68ZpuB26vqFcAs\n",
       "8AdJPAtH0qpTxSeAv5x2HeMwaOS+EdhZVfdV1QHgBuDieW2+AazrvV4HPFRVB8dbpiRNzFPAFQnv\n",
       "nnYhSzFohH0asKtvfjfwQ/PaXAt8Msn9wAuAN4+vPEmauP9Gl41vSjgT2Et364KDwP8D1lWt/DtQ\n",
       "Dgr3YY62/iZwR1XNJjkL+Ksk51fV3vkNk2zpm91WVduGrlSSJqCKxxM+ChwPnEj3tKdZ4ELgADCT\n",
       "cHIVjy7H9pPM9ra3JIPCfQ+wvm9+Pd3ovd9rgN8GqKp7knwNOBu4bf7KqmrLyJVK0oRU8QXgC73Z\n",
       "XwBIeBHwKPAAy3jBU2/Qu+3QfJIrR1nPoH3utwEbkpyR5FjgEmDrvDY76G7KQ5JT6YL93lGKkaSV\n",
       "qoqHq1g1xxMXHLlX1cEkm4GbgRnguqranuSy3vvXAL8DXJ/kTro/Fu+pqoeXuW5J0gK8iEmSFiHh\n",
       "IeDXgA8D3wHsqeLp5dveMl3EJEl6lr8A/hTYB3wd2J/wTwm/NNWq5nHkLkmLlPB8uvPh19PdXfLN\n",
       "wKNVXDH+bTlyl6SJqOLxKg5UcW8VnwVW3HFGbxMgSWOScBLdM1yfqOKr06zFkbskLd2TdBd07gI+\n",
       "DrxvuuUY7pI0Dn9It+99HfA24OyEKxOOSnjBNArygKokjVEvzH8Z+I90B11ngBOreGq09Y2WnYa7\n",
       "JI1ZQoCX0j2M+yHgpCr2j7au0bLTA6qSNGZVFHAPQKY0pHWfuyQtv83JZAfThrskLa+rgC10z8eY\n",
       "GPe5S9IyS/gHuvPfX17FjsV9r1eoStJK9YPAl2Fyp0Ua7pK0zHpPbZroveANd0manC0JL53Ehgx3\n",
       "SZqM/wycAZwziY0Z7pI0AVVsBe6b1PYMd0lqkOEuSQ0y3CWpQYa7JE3WO5LlP6hquEvS5LwP+B7g\n",
       "wuXekOEuSRNSxceB2yexLcNdkhpkuEtSgwx3SZq8dyT8bcIvLNcGfBKTJE3WtcDL6e4U+fKEmSqe\n",
       "HvdGDHdJmqAqPgt8NuEo4L8D6xM+DOyt4sZxbceHdUjSFPTC/Q3A79E9RPvkKs5/brvRstNwl6Qp\n",
       "Szgf+BDw2ioefPZ7PolJklarvcBZwG3jWqHhLklTVsW9wEuB48a1TsNdklaGse4jN9wlqUGGuyQ1\n",
       "aGC4J9mUZEeSu5NcfoQ2s0luT/LlJNvGXqUkaVEWvIgpyQxwFfB6YA/w+SRbq2p7X5uTgD8B3lBV\n",
       "u5OcspwFS5IGGzRy3wjsrKr7quoAcANw8bw2bwX+vKp2A1TVg0iSRnFswqvHsaJB4X4asKtvfndv\n",
       "Wb8NwIuSfCrJbUl+ahyFSdIasxf4OvDpcaxs0L1lhjk15xjgVcDrgBOAzyW5paruXmpxkrRWVPFY\n",
       "wgXAU+NY36Bw3wOs75tfTzd677cLeLCq9gH7knwaOB94Trgn2dI3u62qti22YElq26dIfmzLUtey\n",
       "4L1lkhwNfJVuVH4/8HfApfMOqH4f3UHXN9BdXXUrcElVfWXeury3jCQtIGEGeKqKmbllo2XngiP3\n",
       "qjqYZDNwMzADXFdV25Nc1nv/mqrakeQm4C7g28C184NdkjS8hKOq+PaS1uFdISVpZejdBng/8DTw\n",
       "km4/vHeFlKRVrTda/066gF/STcQMd0laQap4GJa2SwYMd0lqkuEuSQ0y3CWpQYa7JDXIcJeklWlj\n",
       "wsinjxvukrTy3An8L+DsUVdguEvSClPFjwH3ALOjrsNwl6SV6XbgzaN+s7cfkKQVzNsPSJKeYbhL\n",
       "UoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1\n",
       "yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaNDDck2xK\n",
       "siPJ3UkuX6DdBUkOJnnTeEuUJC3WguGeZAa4CtgEnAtcmuScI7T7XeAmIMtQpyRpEQaN3DcCO6vq\n",
       "vqo6ANwAXHyYdu8CPgr885jrkySNYFC4nwbs6pvf3Vv2jCSn0QX+1b1FNbbqJEkjGRTuwwT1e4Ff\n",
       "r6qi2yXjbhlJmrKjB7y/B1jfN7+ebvTe7weAG5IAnAL8eJIDVbV1/sqSbOmb3VZV2xZbsCS1LMks\n",
       "MLvk9XQD7iNu5Gjgq8DrgPuBvwMurartR2h/PfAXVfWxw7xXVeWoXpIWYdTsXHDkXlUHk2wGbgZm\n",
       "gOuqanuSy3rvXzNStZKkZbXgyH2sG3LkLkmLNmp2eoWqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJ\n",
       "apDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QG\n",
       "Ge6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDh\n",
       "LkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho0VLgn2ZRkR5K7k1x+mPffluTOJHcl+UyS88ZfqiRp\n",
       "WKmqhRskM8BXgdcDe4DPA5dW1fa+Nq8GvlJVjyTZBGypqgvnraeqKuP+AJLUslGzc5iR+0ZgZ1Xd\n",
       "V1UHgBuAi/sbVNXnquqR3uytwOmLLUSSND7DhPtpwK6++d29ZUfy88CNSylKkrQ0Rw/RZuH9Nn2S\n",
       "/Cjwc8APH+H9LX2z26pq27DrlqS1IMksMLvU9QwT7nuA9X3z6+lG7/MLOg+4FthUVd883IqqassI\n",
       "NUrSmtEb9G47NJ/kylHWM8xumduADUnOSHIscAmwtb9Bku8GPga8vap2jlKIJGl8Bo7cq+pgks3A\n",
       "zcAMcF1VbU9yWe/9a4ArgJOBq5MAHKiqjctXtiRpIQNPhRzbhjwVUpIWbTlPhZQkrTKGuyQ1yHCX\n",
       "pAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq\n",
       "kOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ\n",
       "7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDQz3JJuS7Ehyd5LLj9Dmj3vv35nk\n",
       "leMvU5K0GAuGe5IZ4CpgE3AucGmSc+a1uQh4WVVtAN4BXL1MtTYjyey0a1gp7Is59sUc+2LpBo3c\n",
       "NwI7q+q+qjoA3ABcPK/NG4EPAFTVrcBJSU4de6VtmZ12ASvI7LQLWEFmp13ACjI77QJWu0Hhfhqw\n",
       "q29+d2/ZoDanL700SdKoBoV7DbmejPh9kqRlcPSA9/cA6/vm19ONzBdqc3pv2XMkMfR7klw57RpW\n",
       "Cvtijn0xx75YmkHhfhuwIckZwP3AJcCl89psBTYDNyS5EPhWVT0wf0VVNX90L0laJguGe1UdTLIZ\n",
       "uBmYAa6rqu1JLuu9f01V3ZjkoiQ7gceBn132qiVJC0qVe0okqTVjv0LVi57mDOqLJG/r9cFdST6T\n",
       "5Lxp1DkJw/xc9NpdkORgkjdNsr5JGfL3YzbJ7Um+nGTbhEucmCF+P05JclOSO3p98TNTKHMikrw/\n",
       "yQNJvrRAm8XlZlWNbaLbdbMTOAM4BrgDOGdem4uAG3uvfwi4ZZw1rJRpyL54NfDC3utNa7kv+tp9\n",
       "EvjfwE9Ou+4p/UycBPw9cHpv/pRp1z3FvtgC/NdD/QA8BBw97dqXqT/+FfBK4EtHeH/RuTnukbsX\n",
       "Pc0Z2BdV9bmqeqQ3eyvtXh8wzM8FwLuAjwL/PMniJmiYfngr8OdVtRugqh6ccI2TMkxffANY13u9\n",
       "Dnioqg5OsMaJqaq/Ab65QJNF5+a4w92LnuYM0xf9fh64cVkrmp6BfZHkNLpf7kO3r2jxYNAwPxMb\n",
       "gBcl+VSS25L81MSqm6xh+uJa4F8muR+4E/jlCdW2Ei06NwedCrlYXvQ0Z+jPlORHgZ8Dfnj5ypmq\n",
       "YfrivcCvV1UlCc/9GWnBMP1wDPAq4HXACcDnktxSVXcva2WTN0xf/CZwR1XNJjkL+Ksk51fV3mWu\n",
       "baVaVG6OO9zHetHTKjdMX9A7iHotsKmqFvq3bDUbpi9+gO5aCej2r/54kgNVtXUyJU7EMP2wC3iw\n",
       "qvYB+5J8GjgfaC3ch+mL1wC/DVBV9yT5GnA23fU3a82ic3Pcu2WeuegpybF0Fz3N/+XcCvw0wEIX\n",
       "PTVgYF8k+W7gY8Dbq2rnFGqclIF9UVUvraozq+pMuv3u72ws2GG4349PAD+SZCbJCXQHz74y4Ton\n",
       "YZi+2AG8HqC3f/ls4N6JVrlyLDo3xzpyLy96esYwfQFcAZwMXN0bsR6oqo3Tqnm5DNkXzRvy92NH\n",
       "kpuAu4BvA9dWVXPhPuTPxO8A1ye5k24g+p6qenhqRS+jJH8GvBY4Jcku4Eq6XXQj56YXMUlSg3zM\n",
       "niQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalB/x9WLe62Ju8XiQAAAABJRU5ErkJg\n",
       "gg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x139154210>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot.plot(recall, precision, label='R-P')\n",
    "\n",
    "plot.xlim([0.0, 1.0])\n",
    "plot.ylim([0.0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def roc_curve(title, Y_test, predicted, label='?'):\n",
    "    fpr, tpr, th = metrics.roc_curve(Y_test, predicted[:,1])\n",
    "    plot.plot(fpr, tpr, label=label);\n",
    "    plot.xlabel('False Positive'); plot.ylabel('True Positive'); plot.title(title)\n",
    "    plot.legend(bbox_to_anchor=(1, 1), loc=2); plot.xlim([0.0, 1.0]); plot.ylim([0.0, 1]);\n",
    "    print 'Logistic Regression (', label, ') AUC = ', metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression ( ? ) AUC =  0.931306706992\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAbQAAAEZCAYAAAD/ttB2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAGuhJREFUeJzt3Xm0ZWV95vHvQwEqEbSwFJVBHBCHVha4KDTisgioiPOA\n",
       "NIg2DgtaxU63scUmRiqJ0di6bLUdgihijAEHMF3aNISopYgMokxGUFAxVKEig4qKpkp+/cfZlzrc\n",
       "usO+VfcMd5/vZ62zuPuc9+zzu5uq89T77ne/O1WFJElL3TajLkCSpMVgoEmSOsFAkyR1goEmSeoE\n",
       "A02S1AkGmiSpEww0LSlJrk9y8KjrkDR+DDQtNdU8lqwkxyT5Q5Lbk/wyyZVJXjCtzT2SvCPJj5P8\n",
       "Nsn3k7xxhn09I8nXkvwqyU1J1iZ5zvB+G2l8GGjSNGkM+GMuqKodgfsCHwD+McnOfa9/FjgIeCZw\n",
       "b+BlwLFJ3tdX54uBzwCnAbtW1QOAtwIGmiaSgaYlq+nFvDfJ+ubxv5Js3/f6m5LcmGRdklcnuTPJ\n",
       "w2bZ19okb0tyAfAb4KFJHpXkvCS3JLkmyeF97e+X5AtND+uS5r3nL6R8gOot1fMPwD2AhzX7Phh4\n",
       "GvCiqvpuVd1ZVRcDRwOvS/KwJnDfA/xVVZ1aVbc3+/taVR27gDqkzjDQtJT9ObAS2Kd5rATeApDk\n",
       "UOC/AQcDewGrmH+o8mjg1fR6RLcA59ELm/sD/xH4UJJHN20/CNwO7AL8J+DlLfa/mSTLgFcAvwC+\n",
       "1zz9NOCiqlrf37aqLgHWNb/T3sBuwOcW+plSVxloWsqOotdDubmqbgb+kt7QHMBLgFOr6uqqugM4\n",
       "iaZXNIsCTmva3wkcCvyoqj7R9JAuB84CDm9C6IXASVX1u6q6GvjEPPuf7olJbgPuAN4FPGeqlwWs\n",
       "AH46y/t+0rx+v75tSRhoWtoeDPy4b/vfmucAHgTc0Pfauhb762//EOCAJLdNPegF6C70AmXbLdh/\n",
       "v4uqajmwHFgDnND32s+b+mfyIODm5sEc7aSJY6BpKbsR2LNvew9gapjuJ8Dufa/1/zyb/iHDfwO+\n",
       "WlXL+x47VtXr6IXJxi3Y/+YfWPUb4DXAU5M8tXn6X+iF6W79bZMc0HzOl4Hv0wvUF2/J50pdZKBp\n",
       "KTsdeEuSFUlW0Jvh9w/Na58BXtFM7NgB+IsW++sfMvwi8MgkRyfZrnnsn+RRVfUHesOPq5PcK8mj\n",
       "6A113hWIzSSTk9r8ElV1G/AR4M3N9peALwFnJnlMkmVJngh8EvhQVf2gmUzyBuAvmssAdkqyTZID\n",
       "k5zc5nOlrjHQtJS9DbgUuLJ5XNo8R1WdA7wf+Aq93syFzXt+P8f+7gqkqvo18HR6k0HW0+vxvQOY\n",
       "mkV5PHAfeue6PkEvXP+9b1+7AV+f43OmTyB5L3BQksc32y9qaj+H3uSTTwIfrarX99V4JnAE8Mqm\n",
       "xp8CfwX80xy/o9RZGeQNPpOcCjwLuKmqHjdLm/fTu9bmt8AxVXXZwArSxGpmJ14FbN9M+ljs/b8T\n",
       "eEBVvaIZKjyjqg5c7M+RNLtB99A+Tm+22IySHAY8oqr2Ao4FPjzgejRBkryguVZtOfBOYM1ihVmS\n",
       "vZM8vrkGeyW9XtLnAapqnWEmDd9AA62qzgdum6PJc+kN19BcOHrfJLsMsiZNlGOBnwHXARvoTb5Y\n",
       "LDsCZwK/Bs4A3l1VaxZx/5IWaNsRf/6ubD71eTd6X0LSVqmqZw5w35fSu2Bb0pgYh0kh0y9GXdIL\n",
       "z0qSRmPUPbT13P36nd3YdB3RXZIYcpK0Bapq0Attj41RB9oaetOfz2ius/lFVc043DhJ/1PmkmR1\n",
       "Va0edR3jwGOxSReORcKt9FZO2Uqrm8eiuq2KnedvNl4mrTMw0EBLcjrwVGBFkhvorae3HUBVnVxV\n",
       "Zyc5LMl19FY4f8Ug65GWsrm/8E8iodWF3GPstqoFrYc5o+QvV1etXr0I9WiJGWigVdWRLdocP8ga\n",
       "pKWgZe9k1i98v8Sl0Q85auHWjrqAMbJ21AUsouVb2TtZu1iFdMDaUReg0RjoSiGLJUl5Dk1LzQLP\n",
       "CS3JczQab5P23WkPTRqcre11SQPRhckiMwW1gSbNYJFm3M21So40Uku55zZbIBtoGmuLN5V7wRZl\n",
       "xp2k4THQNFZmCDCDRVIrBpoGbgsmRxhgkhbMQNPA9AWZISVp4MZhcWJ1QMKtCdX/AKgiTkeX1EaS\n",
       "+yT5+yTrk9zS3AC6NQNNi2V5E179D4NM0kLsDHwLeDTwGODZSQ5v+2aHHDWnBZz/coq6pK1SVT8C\n",
       "3tds/irJ94EHtH2/gab5eHGwpKFL8mJgf+BVbd9joGlWTe/Mnpc0YabOgW+tLf3HcJIDgY8Az66q\n",
       "ze6RORsDTTNqwgzPg0mTZwxGZV4LvKeqvrGQNzkpZELMNAtxrgcYZpJG5oHAjQt9k6vtTwB7W9Jk\n",
       "mu27c9y/U5MsB+6oqt/N8vqM9dtDmwzLDTNJS8ingNbT9afYQ+s4e2fS5FqqPbT52EObIP3ny8Aw\n",
       "kzQZnOW4hM1x0bNrJ0qaOAbaEjFLeBlcktQw0MaQ4SVJC2egjSeXm5KkBXJSyBiZmsyBy01J0oLZ\n",
       "QxuxacOLDitKGook43/N1gIZaCPUd42YISZpaJbyNWhzMdBGy3NlkrRIPIc2It6aRZIWlz200bF3\n",
       "JkmLyB6aJKkTDLQhmOleZDjcKEmLykAboL7ryqgi0x4uGCxJi8hzaIPleTJJGhIDbSvMsdr9FIcV\n",
       "JWlIDLStYw9MksaE59BamGlShxM7JGm8DDTQkhya5Jok1yY5YYbXVyQ5J8nlSb6T5JhB1rNQ80zq\n",
       "cGKHJI2RVA1mfcoky4DvAYcA64FvAkdW1dV9bVYD96iq/5FkRdN+l6raOG1fNay1x2ZYLNjQkrQk\n",
       "DfO7cxwMsoe2Eriuqq6vqg3AGcDzprX5CbBT8/NOwC3Tw2yY+hcLtgcmSUvLICeF7Arc0Le9Djhg\n",
       "WptTgC8nuRHYEXjJAOuZU1+YGWKStAQNMtDajGWeCFxeVauSPBw4L8k+VXX79IbN8OSUtVW1dnHK\n",
       "NMwkdUOSVcCqEZcxMoMMtPXA7n3bu9PrpfX7Y+BvAKrqB0l+BOwNXDp9Z1W1ehBFGmaSuqL5h/7a\n",
       "qe0kJ42smBEY5Dm0S4G9kuyZZHvgCGDNtDbX0Js0QpJd6IXZDwdY090YZpLUHQProVXVxiTHA+cC\n",
       "y4CPVdXVSY5rXj8ZeDvw8SRX0AvXN1XVrYOqaQZeGC1JHTGwafuLaVBTTxPKQJPUVU7blyRpCZq4\n",
       "QOtfxgqXrpKkzpioxYn7L5wedS2SpMU1UYGGk0AkqbMmbshRktRNExNozXCj58wkqaMmacjR4UZJ\n",
       "6rCJ6aFJkrrNQJMkdcJEBJrnzySp+yblHJrnzySp4yaihyZJ6j4DTZLUCQaaJKkTDDRJUicYaJKk\n",
       "TjDQJEmd0OlAm7r3GV6DJkmd1/Xr0Lz+TJImRKd7aJKkyWGgSZI6wUCTJHWCgSZJ6gQDTZLUCQaa\n",
       "JKkTDDRJUicYaJKkTjDQJEmd0NlAS7gVl7ySpInR5aWvXPZKkiZIZ3tokqTJYqBJkjqhVaAl2SHJ\n",
       "3oMuRpKkLTVvoCV5LnAZcG6zvW+SNYMubEtM3f/Me6BJ0uRp00NbDRxAExBVdRnwsAHWtDWWV5Hm\n",
       "sfOoi5EkDU+bQNtQVb+Y9tydbXae5NAk1yS5NskJs7RZleSyJN9JsrbNfmfej9P0JWmStZm2/69J\n",
       "Xgpsm2Qv4L8A35jvTUmWAR8ADgHWA99Msqaqru5rc1/gg8AzqmpdkhVb8ks0nKYvSROsTQ/t9cBj\n",
       "gd8DpwO/Av5ri/etBK6rquuragNwBvC8aW2OAs6sqnUAVXVz28IlSerXpoe2d1WdCJy4wH3vCtzQ\n",
       "t72O3rm4fnsB2yX5CrAj8L6q+uQCP8fhRklSq0B7T5IHAp8FPl1V32m572rRZjtgP+BgYAfgwiQX\n",
       "VdW1LT9jisONkjTh5g20qlqV5EHAS4CTk+wEfKaq/nqet64Hdu/b3p1eL63fDcDNVXUHcEeSrwH7\n",
       "AJsFWpLVfZtrq2rtfLVL0iRJsgpYNeIyRiZVbTpSTePkccAJwBFVtd08bbcFvkev93UjcAlw5LRJ\n",
       "IY+iN3HkGcA9gIubfX932r6qqmbtgSWUPTRJurv5vju7Zt4eWpLH0OudvRi4Bfg08Ib53ldVG5Mc\n",
       "T++C7GXAx6rq6iTHNa+fXFXXJDkHuJLepQCnTA8zSZLamLeHluQiejMUP1tV64dS1eY12EOTpAWa\n",
       "tB7agoYcR2Wu/ynNDEdcGUSS7m7SAm3WIcckn62qw5NcNcPLVVWPH2BdC+EMR0nS7D20JA+uqhuT\n",
       "PAQ2C4yqqh8PvLpNtczVQ3O4UZJmMGk9tFlXCqmqG5sfX9us9nHXA3jtUKqTJKmlNktfPX2G5w5b\n",
       "7EIkSdoac51Dew29ntjDp51H2xG4YNCFteGSV5KkKXOdQ7sPsBz4W3oXU0+Nw95eVbcMp7y7aplx\n",
       "HNjzZ5I0u0k7hzZXoO1UVb9Kcj9mWJexqm4ddHF9tRhokrRAkxZoc60UcjrwLOBbzLzQ8EMHUlFL\n",
       "DjdKkvot2Qur7Z1J0twmrYc27yzHJE9Ocu/m55cleU9zbZokSWOjzbT9vwN+m2QfeosS/xD4+4FW\n",
       "JUnSArUJtI1VdSfwfOCDVfUBelP3JUkaG23uWH17khOBo4GnJFlG707TkiSNjTY9tCOA3wOvrKqf\n",
       "ArsC7xpoVZIkLVCrWY5JHgjsT2/6/iVVddOgC5v2+c5ylKQFcpbjNEleAlwMHE7vztWXJDl80IVJ\n",
       "krQQbe5YfSVwyFSvLMn9gS8N835o9tAkaeHsoW0uwM/7tm9h8/ujSZI0Um1mOZ4DnJvkH+kF2RHA\n",
       "/xtoVZIkLVDbSSEvBA5sNs+vqs8PtKrNP98hR0laoEkbcpzrfmiPpDc9/xHAlcB/r6p1wypMkqSF\n",
       "mOsc2qnAF4EXAd8G3j+UiuaRcGtC4Ur7kqQ+c51Du3dVndL8fE2Sy4ZRUAvLHWqUJE03V6DdM8l+\n",
       "zc8B7tVsB6iq+vbAq5MkqaW57li9lrvf2DP921V10EAru3std53YdDKIJLUzaZNCltwNPg00SWpn\n",
       "0gKtzYXVkiSNvSUVaAm34uxGSdIM2qwUMk6c4ShJmlGb1fa3SfKyJG9ttvdIsnLwpUmS1F6bIccP\n",
       "AU8Cjmq2f908J0nS2Ggz5HhAVe07dWF1Vd2aZLsB1yVJ0oK06aH9e5JlUxvN/dDuHFxJkiQtXJtA\n",
       "+9/A54EHJHk7cAHwjoFWJUnSArW9fcyjgYObzS9V1dUDrWrzz6+qihdVS1J7Xlg9TZI9gN8AX2ge\n",
       "v2mem1eSQ5Nck+TaJCfM0W7/JBub+65JkrRgbSaFnM2mNRzvCTwU+B7w2Lne1Jx3+wBwCLAe+GaS\n",
       "NdN7d027d9K7M/bE/EtCkrS45g20qvoP/dvNivuva7HvlcB1VXV9874zgOcB04crXw98Dti/xT4l\n",
       "SZrRgpe+am4bc0CLprsCN/Rtr2ueu0uSXemF3Iendr/QeiRJghY9tCR/1re5DbAfvSHE+bQJp/cC\n",
       "b25mfASHHCVJW6jNObR79/28EfgicGaL960Hdu/b3p1eL63fE4AzelnGCuCZSTZU1ZrpO0uyGk4i\n",
       "+cvVwNqqWtuiBkmaGElWAatGXMbIzDltv5mw8T+r6s9mbTT7e7elN3nkYOBG4BLgyNmm/Cf5OPCF\n",
       "qjprhtecti9JCzRp0/Zn7aEl2baqNiZ5cpqjspAdN+89HjgXWAZ8rKquTnJc8/rJW1W5JEl9Zu2h\n",
       "Jfl2Ve2X5O+ABwOfBX7bvFwz9aQGxR6aJC2cPbRNpg7CPYFbgD+Z9vrQAk2SpPnMFWj3T/IG4Kph\n",
       "FSNJ0paaK9CWATsOqxBJkrbGXOfQLquqfYdcz4w8hyZJCzdp59AWvFKIJEnjaK5AO2RoVUiStJVm\n",
       "DbSqumWYhUiStDUccpQkdYKBJknqBANNktQJBpokqRMMNElSJxhokqROMNAkSZ1goEmSOsFAkyR1\n",
       "goEmSeoEA02S1AkGmiSpEww0SVInLJlAS7gVuG3UdUiSxtOsd6weJ0kKCu9WLUntecdqSZKWIANN\n",
       "ktQJBpokqRMMNElSJxhokqROMNAkSZ1goEmSOsFAkyR1goEmSeoEA02S1AkGmiSpEww0SVInGGiS\n",
       "pE4w0CRJnTDwQEtyaJJrklyb5IQZXn9pkiuSXJnkgiSPH3RNkqTuGej90JIsA74HHAKsB74JHFlV\n",
       "V/e1eRLw3ar6ZZJDgdVV9cRp+/F+aJK0QN4PbXGtBK6rquuragNwBvC8/gZVdWFV/bLZvBjYbZZ9\n",
       "ebdqSdKsBh1ouwI39G2va56bzauAs2d6oYqdF7EuSVLHbDvg/bcez0xyEPBK4MmzvL66b3NtVa3d\n",
       "qsokqWOSrAJWjbiMkRl0oK0Hdu/b3p1eL+1umokgpwCHVtWMQ4tVtXoQBUpSVzT/0F87tZ3kpJEV\n",
       "MwKDHnK8FNgryZ5JtgeOANb0N0iyB3AWcHRVXTfgeiRJHTXQHlpVbUxyPHAusAz4WFVdneS45vWT\n",
       "gbcCy4EPJwHYUFUrB1mXJKl7Bjptf7FM2tRTSVoMk/bd6UohkqROMNAkSZ1goEmSOsFAkyR1goEm\n",
       "SeoEA02S1AkGmiSpEww0SVInGGiSpE4w0CRJnWCgSZI6wUCTJHWCgSZJ6gQDTZLUCQaaJKkTDDRJ\n",
       "UicYaJKkTjDQJEmdYKBJkjrBQJMkdYKBJknqBANNktQJBpokqRMMNElSJxhokqROMNAkSZ1goEmS\n",
       "OsFAkyR1goEmSeoEA02S1AkGmiSpEww0SVInGGiSpE4w0CRJnWCgSZI6YaCBluTQJNckuTbJCbO0\n",
       "eX/z+hVJ9h1kPZKk7hpYoCVZBnwAOBR4DHBkkkdPa3MY8Iiq2gs4FvjwoOrpiiSrRl3DuPBYbOKx\n",
       "2MRjMbkG2UNbCVxXVddX1QbgDOB509o8F/gEQFVdDNw3yS4DrKkLVo26gDGyatQFjJFVoy5gjKwa\n",
       "dQEajUEG2q7ADX3b65rn5muz2wBrkiR11CADrVq2yxa+T5Kku2w7wH2vB3bv296dXg9srja7Nc9t\n",
       "JolB10hy0qhrGBcei008Fpt4LCbTIAPtUmCvJHsCNwJHAEdOa7MGOB44I8kTgV9U1c+m76iqpvfi\n",
       "JEm6m4EFWlVtTHI8cC6wDPhYVV2d5Ljm9ZOr6uwkhyW5DvgN8IpB1SNJ6rZUOZInSVr6xmqlEC/E\n",
       "3mS+Y5Hkpc0xuDLJBUkeP4o6h6HNn4um3f5JNiZ54TDrG5aWfz9WJbksyXeSrB1yiUPT4u/HiiTn\n",
       "JLm8ORbHjKDMoUhyapKfJblqjjYT8b1JVY3Fg96w5HXAnsB2wOXAo6e1OQw4u/n5AOCiUdc9wmPx\n",
       "JOA+zc+HTvKx6Gv3ZeCLwItGXfeI/kzcF/hXYLdme8Wo6x7hsVgNvGPqOAC3ANuOuvYBHY+nAPsC\n",
       "V83y+kR8b1bVWPXQvBB7k3mPRVVdWFW/bDYvprvX77X5cwHweuBzwM+HWdwQtTkORwFnVtU6gKq6\n",
       "ecg1DkubY/ETYKfm552AW6pq4xBrHJqqOh+4bY4mk/K9OVaB5oXYm7Q5Fv1eBZw90IpGZ95jkWRX\n",
       "el9oU0undfHEcJs/E3sBOyf5SpJLk7xsaNUNV5tjcQrw2CQ3AlcAfzqk2sbRpHxvDnTa/kJ5IfYm\n",
       "rX+nJAcBrwSePLhyRqrNsXgv8OaqqiRh8z8jXdDmOGwH7AccDOwAXJjkoqq6dqCVDV+bY3EicHlV\n",
       "rUrycOC8JPtU1e0Drm1cTcL35lgF2qJeiL3EtTkWNBNBTgEOraq5hhyWsjbH4gn0rmWE3vmSZybZ\n",
       "UFVrhlPiULQ5DjcAN1fVHcAdSb4G7AN0LdDaHIs/Bv4GoKp+kORHwN70ro+dNJPyvTlWQ453XYid\n",
       "ZHt6F2JP/0JaA7wcYK4LsTtg3mORZA/gLODoqrpuBDUOy7zHoqoeVlUPraqH0juP9pqOhRm0+/vx\n",
       "f4ADkyxLsgO9CQDfHXKdw9DmWFwDHALQnC/aG/jhUKscH5PyvTk+PbTyQuy7tDkWwFuB5cCHm57J\n",
       "hqpaOaqaB6Xlsei8ln8/rklyDnAlcCdwSlV1LtBa/pl4O/DxJFfQ+4f7m6rq1pEVPUBJTgeeCqxI\n",
       "cgNwEr3h54n63gQvrJYkdcQ4DTlKkrTFDDRJUicYaJKkTjDQJEmdYKBJkjrBQJMkdYKBpiUtyR+a\n",
       "26VMPfaYo+2vF+HzTkvyw+azvtVcqLrQfZyS5FHNzydOe+2Cra1RmlReh6YlLcntVbXjYredYx8f\n",
       "B75QVWcleRrw7qraZyv2t9U1Seqxh6ZOSfJHSf6l6T1dmeS5M7R5UJKvNb2sq5Ic2Dz/9CTfaN77\n",
       "mSR/NNvHNP89H3hE8943NPu6Ksmf9tXyf5ubTF6V5PDm+bVJnpDkb4F7NXV8snnt181/z0hyWF/N\n",
       "pyV5YZJtkrwrySXNzRqPXZwjJy19Y7P0lbSF7pXksubnHwIvAV5QVbcnWQFcyObr/B0FnFNVb0+y\n",
       "DbBD0/bPgYOr6o707oL8BuCv5/js5wBXJtkPOIbefbq2AS5O8lXg4cD6qnoWQJKp+3MVUFX15iSv\n",
       "q6r+OwhPDZmc0fwuZzfrFf4JcBzwanpr8a1Mcg/g60n+uaqub3m8pM4y0LTU3dEfCEm2A96R5Cn0\n",
       "1jN8cJIHVNVNfe+5BDi1aftPVXVFklXAY4BvNGtjbg98Y4bPC/CuJG8BbqJ3L7qnAWc1q9yT5Cx6\n",
       "dxE+B3h30xP7YlV9fQG/1znA+5oweybw1ar6fZKnA49L8uKm3U70eonXL2DfUicZaOqal9K7hcx+\n",
       "VfWH5rYh9+xvUFXnN4H3bOC0JO+hd8ff86rqqHn2X8Abq+qsqSeSHMLd7zeV3sfUtUn2BZ4FvC3J\n",
       "l6pqrh5ff42/S7IWeAa9ntrpfS8fX1XntdmPNEk8h6au2Qm4qQmzg4CHTG/QzIT8eVV9FPgosC9w\n",
       "EfDk9G4GOXX+a69ZPmP6zRLPB56f5F7NebfnA+cneRDwu6r6FPDu5nOm25Bktn9YfprezVunenvQ\n",
       "W2H+tVPvSfLI5lYx0sSzh6albvo03U8BX0hyJb37Zl09Q9uDgDcm2QDcDry8qm5OcgxwenNuCnrn\n",
       "1Ga6OebdPrOqLktyGr2hTOjdtuWKZnjwXUnuBDYA/3mGfX2E3nm4b1XVy6bt+5+BT9IbFt3YPPdR\n",
       "YE/g2+mNjd4EvGCG/UoTx2n7kqROcMhRktQJBpokqRMMNElSJxhokqROMNAkSZ1goEmSOsFAkyR1\n",
       "goEmSeqE/w+0b+Iif99xhQAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113115a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_curve('log reg, ROC', Y_test, predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_l1 = LogisticRegression(penalty='l1')\n",
    "model_l1.fit(X_train, Y_train)\n",
    "predict_l1 = model_l1.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression ( l2 ) AUC =  0.931306706992\n",
      "Logistic Regression ( l1 ) AUC =  0.91215975805\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAbgAAAEZCAYAAADlijD4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAHS5JREFUeJzt3X3YZXVd7/H3hwEVE3KI9DLA8IFQK7mkI3jSjkP4MGKJ\n",
       "zx58KB+69JRY1zELD5ozZGkevTzm0Qzx8ZgyaVIHPYSZOkkoEIqANShkU8ygITAWKhrI9/yx1j1s\n",
       "7rkf1v2w7r3vtd+v69rX3Gvvtdf+3othf+b3W7/fb6WqkCRpaPYbdwGSJPXBgJMkDZIBJ0kaJANO\n",
       "kjRIBpwkaZAMOEnSIBlwWleS7Exy4rjrkDT5DDitN9U+1q0kz0/ygyQ3J/m3JFckecqsfe6a5PVJ\n",
       "/jnJd5N8Nckr5jjW45N8Nsm/J7k+yfYkv7h2v400uQw4aZa0ev6YC6vqIOCewNuADyU5ZOT1jwAn\n",
       "AE8A7gE8D3hxkj8cqfPpwIeB9wGHVdW9gNcABpyEAad1rG3lvCXJ7vbxv5LcZeT1305yXZJdSX4l\n",
       "ye1J7j/PsbYn+b0kFwLfAe6X5EFJPpnkxiRXJXnGyP4/kuRjbQvskva9FyylfIBqlhL6E+CuwP3b\n",
       "Y58IPBZ4WlX9Q1XdXlUXA88FXprk/m0Avxn43ap6T1Xd3B7vs1X14iXUIQ2WAaf17FXAccAx7eM4\n",
       "4NUASTYD/x04ETgK2MTiXZvPBX6FpsV0I/BJmvD5UeC/An+U5MHtvm8HbgbuDfwy8Esdjr+PJBuA\n",
       "FwDfAr7SPv1Y4KKq2j26b1VdAuxqf6ejgcOBP1vqZ0rTwoDTevZsmhbMDVV1A3AGTVcewDOB91TV\n",
       "jqq6BdhC22qaRwHva/e/HdgM/FNVvb9tQX0JOAd4RhtKTwW2VNX3qmoH8P5Fjj/bI5LsAW4B3gj8\n",
       "4kwrDDgU+MY87/t6+/qPjGxLmoMBp/Xsx4B/Htn+l/Y5gPsA1468tqvD8Ub3/3Hg+CR7Zh40gXpv\n",
       "moDZfxnHH3VRVW0ENgLnAqeNvPbNtv653Ae4oX2wwH7S1DPgtJ5dBxw5sn1fYKZb7+vAESOvjf48\n",
       "n9Euxn8B/qaqNo48Dqqql9KEy23LOP6+H1j1HeBXgUcneXT79F/ThOvho/smOb79nE8DX6UJ2Kcv\n",
       "53OlaWDAaT07G3h1kkOTHEozgvBP2tc+DLygHShyd+B3OhxvtIvx48BPJHlukgPax8OTPKiqfkDT\n",
       "Xbk1yYFJHkTTNbo3INtBK1u6/BJVtQd4J/DKdvtTwKeAjyZ5SJINSR4BfAD4o6r6x3ZwysuB32mn\n",
       "HRycZL8kj0pyZpfPlYbOgNN69nvApcAV7ePS9jmq6nzgrcBnaFo7n2/f8/0Fjrc3oKrq28DjaAaX\n",
       "7KZpEb4emBmleSrwwzTXyt5PE7b/MXKsw4G/XeBzZg9IeQtwQpKHtttPa2s/n2YwyweAd1XVy0Zq\n",
       "/CjwLOCFbY3fAH4X+IsFfkdpaqTPG54meQ/wROD6qvrpefZ5K81cn+8Cz6+qy3orSFOrHf14JXCX\n",
       "dhDJah//DcC9quoFbdfitqp61Gp/jqTu+m7BvZdmNNqckpwEPLCqjgJeDLyj53o0RZI8pZ0rtxF4\n",
       "A3DuaoVbkqOTPLSdE34cTSvqzwGqapfhJo1frwFXVRcAexbY5Uk03Tu0E1nvmeTefdakqfJi4F+B\n",
       "a4BbaQZzrJaDgI8C3wa2AW+qqnNX8fiSVmj/MX/+Yew71Ppwmi8laUWq6gk9HvtSmgnkkibUJAwy\n",
       "mT05dl0vpCtJmgzjbsHt5s7zhw7njnlMeyUx9CRpGaqq74XDJ9a4A+5cmuHW29p5Pt+qqjm7J6f5\n",
       "P9KoJFurauu465gEnos7DOFcJNwEbOS0Q+DAhS7dz+GWjfCGm9qNre1jVe2p4pDFd5ss09446DXg\n",
       "kpwNPBo4NMm1NOsBHgBQVWdW1XlJTkpyDc0K7i/osx5pPdsbAHPaQkKnieVjs1hwbd37057aUksP\n",
       "kz9o/kjO2Fq1deuC+2oq9BpwVXVKh31O7bMGaT1YOLz22lM194LO4/hSzxnpUvOo5QWXtEzj7qLU\n",
       "0m0fdwETZPu4C1hFG+cLr462r1YhixkJtj21ZSIvHWwfdwGaDL2uZLJakpTX4LTedGyVzej9Gs8y\n",
       "WlzzsSW2Tkz7d6ctOKk/K22VzWkFQTWpLS6tsSEOPpkryG3BSXPo3Ppazoi/lbMFpU7m++4c2nfq\n",
       "vL+nAadJtsRuvqVZnXAybDSxpj3g7KLURJkj0OYdObjsz7jzIAnDSRooA069W8Zgi77/ZbnRa1HS\n",
       "8Blw6s1IsK1FaM1fx76DMtb8opmkxSXZCbyovav9ihlwWhXztNLGGmywN9ywxSatCwVUkl8Gfh14\n",
       "IPDvwIeA06vqB0s5mAGn1bJqQ+JXcb4WeJ1NWm8CHEgTcBcD96JZt/gVNDcu7syA04KWcP1szm6/\n",
       "ZYaV87Wk6VVV9ccj29cl+SBwwlIPZMBpMctumdk9KGmVPBr48lLfZMBpXm3rrdOAjHlaanYPSutQ\n",
       "sjo3nl6NyxZJXggcC7xwqe814DSnNtzosj6iLTVpWMY9OGxGkicDrwNOrKqbFtt/NgNuSixjRZBF\n",
       "F/91wrSkviTZDLwTOKmq/n45xzDgpsBIa2y1Rzk6GETSqkvy88AHgZOr6tLlHseAmw6rPYTf7khJ\n",
       "fQnwKuAg4C+TvV81n62qJy7lQAbcwC1loEhHLnMlqRdVdb/2x1VZyWS/1TiIJkvCTQk1MxKq7xtp\n",
       "StIksgW3ji0wcGTsS2RJ0rgZcOvEWq71uMjqIy5ULGldMOAm0AQsXOx1NknrngE3mVZt1ON8bKVJ\n",
       "GjoDboKM3j9tWe9f2sLGzmGTNGgG3JjN6o5caTekXYuS1DLgxmg1VhiZ1Wqza1GSWgbceK3GtTZb\n",
       "bZIGIclO4EVVtSoTvQ24Mem6wkiH62q22iQNRQGV5CeBN9PcJudHqmpZi5IYcOOzYOvNBY0lTbFb\n",
       "gW3A24G/WO5BDLjJZdejpKlUVV8FvprkgSs5jgG3BvaZuH3aIbB1Dzljwbvm2vUoSStgwPVodF7b\n",
       "aHdkzthTts4kTaqckYX+8d3ZuL/nDLh+9b4iiSSttnEH02ox4FZggdX8Z+yBOUdC2v0oST0z4FZm\n",
       "0Raad8CWpKVLcjfgLu3PdwWoqu8v5Rje8LSD0RuIjj5YpCU2Em7ecFSSOkpyJPBd4Ms0c+NuAXYs\n",
       "9Ti9BlySzUmuSnJ1ktPmeP3QJOcn+VKSLyd5fp/1LNVMsEGznNYcj3mDy3CTpKWpqvtV1aeramdV\n",
       "7dc+NrR/3n+px0vVqgyW2ffAyQbgK8BjgN3A3wGnVNWOkX22Anetqv+R5NB2/3tX1W2zjlVVa9PF\n",
       "N8fix50Caq7rbIabpHGa77tzLb9T18J8v0+f1+COA66pqp1tAduAk7lzM/PrwEPbnw8Gbpwdbmtp\n",
       "OYsfu+KIJE2mPgPuMODake1dwPGz9jkL+HSS64CDgGf2WM+CRsJtqa0uVxyRpAnUZ8B16fs8HfhS\n",
       "VW1K8gDgk0mOqaqbZ+/YdmfO2F5V21enzBWFmyRNjCSbgE1jLmNi9Blwu4EjRraPoGnFjfpZ4PcB\n",
       "quofk/wTcDRw6eyDVdXWPopcSbi13ZPOaZM0Edp/+G+f2U6yZWzFTIA+R1FeChyV5MgkdwGeBZw7\n",
       "a5+raAahkOTeNOH2tR5rupNVaLltdCCJJE2m3lpwVXVbklOBTwAbgHdX1Y4kL2lfPxN4HfDeJJfT\n",
       "hO1vV9VNfdU0B5fSkjSVktVZb3KS9TZNYDX1NaQ1oVYScDkjLposaWINbTrAUrmSiSRpkKYu4EaX\n",
       "3WIFA0QcYCJJk22qFltezkTuBTj/TZIm2FQFHA4qkaSpMW0BtyxzrDMJdk9K0kSbmoBruyeXG0p2\n",
       "R0rSOjM1AYfdk5I0VaZuFKUkaToYcJKkQZqKgFvJ9Tfnu0nS+jQt1+BWcv3NASaStA5NRQtuuWy9\n",
       "SdL6ZcDNow03vB2OJK1P09JFuRx2TUrSOmbAzTKyaoldk5K0jhlw+7LlJkkD4DU4SdIgDTrgZu79\n",
       "RsfuRkdNStJwDL2Lcqnz3+yelKSBGHQLTpI0vYbeguvEkZOSNDwGXMOuSUkaGLsoJUmDZMBJkgZp\n",
       "6gPOqQGSNExTHXAuqCxJwzW1AWe4SdKwTd0oytEpAYabJA3XYAMuYb5ra04JkKQpMNiAY+nLdEmS\n",
       "BmTIAXcnrlYiSdNlagIOuyYlaap0GkWZ5O5Jju67mL44102Sps+iAZfkScBlwCfa7YclObfvwpZj\n",
       "5v5vo/eAczqAJE2nLi24rcDxtIFRVZcB9++xppXYWEXaxyF7nzPcJGnqdAm4W6vqW7Oeu73LwZNs\n",
       "TnJVkquTnDbPPpuSXJbky0m2dznu3MfZd1qAXZOSNL26DDL5+yTPAfZPchTw68DnFntTkg3A24DH\n",
       "ALuBv0tyblXtGNnnnsDbgcdX1a4khy7nl2htZGv25Axq5Dknc0vSlOoScC8DXgV8Hzib5lrcazu8\n",
       "7zjgmqraCZBkG3AysGNkn2cDH62qXQBVdUPnyufmSElJEtAt4I6uqtOB05d47MOAa0e2d9Fcyxt1\n",
       "FHBAks8ABwF/WFUfWOLnjHZPblzqeyVJw9TlGtyb2+tor03yU0s4di2+CwcAxwInAY8HfqftBl2q\n",
       "jSODSiRJWrwFV1WbktwHeCZwZpKDgQ9X1WLdlLuBI0a2j6BpxY26Frihqm4BbknyWeAY4OrZB0uy\n",
       "dWRze1VtX6x2SZomSTYBm8ZcxsRIVZeGVrtz8tPAacCzquqARfbdH/gKcCJwHXAJcMqsQSYPohmI\n",
       "8njgrsDF7bH/Ydaxqmr+a2sJxdY00xgcVCJJwOLfnUO3aAsuyUNoWm9PB24E/hR4+WLvq6rbkpxK\n",
       "MyhlA/DuqtqR5CXt62dW1VVJzgeuoJl6cNbscFsCB5hIkvZatAWX5CJgG/CRqtq9JlXtW0OXFhwG\n",
       "nCTdwRbcIqrqEWtRyHI5glKSNJd5R1Em+Uj755VzPK5YuxIX5QhKSdI+FmrB/Ub75y/APjcO7T4y\n",
       "ZQ24JJckabZ5W3BVdV37469V1c7RB/Bra1Jddy6oLEm6ky4TvR83x3MnrXYhkiStpnm7KJP8Kk1L\n",
       "7QFJrhx56SDgwr4L68IBJpKk+Sx0De5DwF8Cf0AzuXvmOtzNVXVj34V1tLGKzLqDgCRJCwZcVdXO\n",
       "JC9l1qCSJIdU1U39ltaNA0wkSXNZKODOBp4IfIG5R03er5eKOhrtnnSCtyRptnkDrqqe2P555JpV\n",
       "szR2T0qS5rXoKMokj0xyj/bn5yV5c5If7780SZKWr8s0gT8GvpvkGJpFlr8G/J9eq5IkaYW6BNxt\n",
       "VXU78GTg7VX1NpqpApIkTaxFF1sGbk5yOvBc4OeSbKC5E7ckSROrSwvuWcD3gRdW1TeAw4A39lqV\n",
       "JEkrtGjAVdXXgQ8C90zyC8D3qsprcJKkidZlFOUzgYuBZ9Dc2fuSJM/ouzBJklaiyzW4VwMPr6rr\n",
       "AZL8KPAp4CN9FtaFq5hIkubTJeACfHNk+0b2vT/cuLiKiSRpTl0C7nzgE0k+RBNsz6JZhFmSpIm1\n",
       "aMBV1W8leSrwqPapM6vqz/stS5KklVnofnA/QTMd4IHAFcBvVdWutSpMkqSVWGgU5XuAjwNPA74I\n",
       "vHVNKlpEwk0JhYNLJEkLWCjg7lFVZ1XVVVX1RsZ8e5wRG6tIFYeMuxBJ0uRa6Brc3ZIc2/4c4MB2\n",
       "OzQ3Q/1i79VJkrRMqZr7dmpJtnPnG51mdLuqTui1sjvXUlXNdICEau4Dl5sAakvZkpOkOYx+d06j\n",
       "hW54umkN61gO58BJkubVZbFlSZLWnXUVcAkuzSVJ6qTLSiaTZGPVxCwTJkmaYF3uJrBfkucleU27\n",
       "fd8kx/VfmiRJy9elBfdHwO3AzwO/C3y7fe4/9VjXvNrRkxuxq1KStIAuAXd8VT0syWUAVXVTkgN6\n",
       "rmshjp6UJC2qyyCT/0iyYWajvR/c7f2VJEnSynUJuP8N/DlwrySvAy4EXt9rVZIkrVCX2+X8SZIv\n",
       "ACe2T51cVTv6LUuSpJXpMoryvsB3gI+1j++0zy0qyeYkVyW5OslpC+z38CS3tfedkyRpxboMMjmP\n",
       "O9agvBvNXQW+AvzkQm9qr9u9DXgMsBv4uyTnzm79tfu9gebO4Q4ekSStii5dlD81ut3eUeClHY59\n",
       "HHBNVe1s37cNOBmY3b35MuDPgId3OKYkSZ0seamu9jY5x3fY9TDg2pHtXe1zeyU5jCb03jFz+KXW\n",
       "I0nSXBZtwSX5zZHN/YBjabocF9MlrN4CvLKqKkmwi1KStEq6XIO7x8jPtwEfBz7a4X27gSNGto+g\n",
       "acWN+hlgW5NtHAo8IcmtVXXu7IMl2Qpb4DOQrdlUVds71CBJUyPJJmDTmMuYGPPe8BT2DgD5n1X1\n",
       "m/PuNP9796cZjHIicB1wCXDKfFMMkrwX+FhVnTPHa20jj2JrcCUTSVqcNzydR5L9q+q2JI9Me5aW\n",
       "cuD2vacCnwA2AO+uqh1JXtK+fuaKKpckaQHztuCSfLGqjk3yx8CPAR8Bvtu+XHO1tPpiC06Sls4W\n",
       "3PxmTsrdgBtp7iYwas0CTpKkpVoo4H40ycuBK9eqGEmSVstCAbcBOGitCunktEPA+8BJkjpYKOC+\n",
       "UVVnrFklXRy4h9pSh4y7DEnS5FvySiaSJK0HCwXcY9asCkmSVtm8AVdVN65lIZIkrSa7KCVJg2TA\n",
       "SZIGyYCTJA2SASdJGiQDTpI0SAacJGmQDDhJ0iAZcJKkQTLgJEmDZMBJkgbJgJMkDZIBJ0kaJANO\n",
       "kjRI6ybgEm4adw2SpPVj3QQcsHHcBUiS1o/1FHCSJHVmwEmSBsmAkyQNkgEnSRokA06SNEgGnCRp\n",
       "kAw4SdIgGXCSpEEy4CRJg2TASZIGyYCTJA2SASdJGiQDTpI0SAacJGmQeg+4JJuTXJXk6iSnzfH6\n",
       "c5JcnuSKJBcmeeicBzrtEIA9PZcrSRqIXgMuyQbgbcBm4CHAKUkePGu3rwH/paoeCrwWeOecBztw\n",
       "D7WlDumxXEnSgPTdgjsOuKaqdlbVrcA24OTRHarq81X1b+3mxcDhPdckSZoCfQfcYcC1I9u72ufm\n",
       "8yLgvF4rkiRNhf17Pn513THJCcALgUfOucNnIFuztd3aXlXbV1qcJA1Jkk3ApjGXMTH6DrjdwBEj\n",
       "20fQtOLupB1YchawuarmHkhyAntqe23toUZJGoT2H/7bZ7aTbBlbMRMgVZ0bWUs/eLI/8BXgROA6\n",
       "4BLglKraMbLPfYFPA8+tqovmOU5VVXorVJIGaNq/O3ttwVXVbUlOBT4BbADeXVU7krykff1M4DXA\n",
       "RuAdSQBurarj+qxLkjR8vbbgVsu0/ytEkpZj2r87XclEkjRIBpwkaZAMOEnSIBlwkqRBMuAkSYNk\n",
       "wEmSBsmAkyQNkgEnSRokA06SNEgGnCRpkAw4SdIgGXCSpEEy4CRJg2TASZIGyYCTJA2SASdJGiQD\n",
       "TpI0SAacJGmQDDhJ0iAZcJKkQTLgJEmDZMBJkgbJgJMkDZIBJ0kaJANOkjRIBpwkaZAMOEnSIBlw\n",
       "kqRBMuAkSYNkwEmSBsmAkyQNkgEnSRokA06SNEgGnCRpkAw4SdIg9RpwSTYnuSrJ1UlOm2eft7av\n",
       "X57kYX3WI0maHr0FXJINwNuAzcBDgFOSPHjWPicBD6yqo4AXA+/oq56hSLJp3DVMCs/FHTwXd/Bc\n",
       "aEafLbjjgGuqamdV3QpsA06etc+TgPcDVNXFwD2T3LvHmoZg07gLmCCbxl3ABNk07gImyKZxF6DJ\n",
       "0GfAHQZcO7K9q31usX0O77EmSdKU6DPgquN+Web7JEma1/49Hns3cMTI9hE0LbSF9jm8fW4fSQy+\n",
       "VpIt465hUngu7uC5uIPnQtBvwF0KHJXkSOA64FnAKbP2ORc4FdiW5BHAt6rqX2cfqKpmt/IkSVpQ\n",
       "bwFXVbclORX4BLABeHdV7Ujykvb1M6vqvCQnJbkG+A7wgr7qkSRNl1TZ8ydJGp6JWsnEieF3WOxc\n",
       "JHlOew6uSHJhkoeOo8610OXvRbvfw5PcluSpa1nfWun4/8emJJcl+XKS7Wtc4prp8P/HoUnOT/Kl\n",
       "9lw8fwxlrokk70nyr0muXGCfqfje3EdVTcSDphvzGuBI4ADgS8CDZ+1zEnBe+/PxwEXjrnuM5+I/\n",
       "Az/c/rx5ms/FyH6fBj4OPG3cdY/p78Q9gb8HDm+3Dx133WM8F1uB18+cB+BGYP9x197T+fg54GHA\n",
       "lfO8PhXfm3M9JqkF58TwOyx6Lqrq81X1b+3mxQx3/mCXvxcALwP+DPjmWha3hrqch2cDH62qXQBV\n",
       "dcMa17hWupyLrwMHtz8fDNxYVbetYY1rpqouAPYssMu0fG/uY5ICzonhd+hyLka9CDiv14rGZ9Fz\n",
       "keQwmi+4maXehnhhucvfiaOAQ5J8JsmlSZ63ZtWtrS7n4izgJ5NcB1wO/MYa1TaJpuV7cx99ThNY\n",
       "KieG36Hz75TkBOCFwCP7K2esupyLtwCvrKpKEvb9OzIEXc7DAcCxwInA3YHPJ7moqq7utbK11+Vc\n",
       "nA58qao2JXkA8Mkkx1TVzT3XNqmm4XtzH5MUcKs6MXyd63IuaAeWnAVsrqqFuijWsy7n4mdo5lJC\n",
       "c73lCUlurapz16bENdHlPFwL3FBVtwC3JPkscAwwtIDrci5+Fvh9gKr6xyT/BBxNMz932kzL9+Y+\n",
       "JqmLcu/E8CR3oZkYPvsL6lzglwAWmhg+AIueiyT3Bc4BnltV14yhxrWy6LmoqvtX1f2q6n401+F+\n",
       "dWDhBt3+//i/wKOSbEhyd5oBBf+wxnWuhS7n4irgMQDt9aajga+taZWTY1q+N/cxMS24cmL4Xl3O\n",
       "BfAaYCPwjrblcmtVHTeumvvS8VwMXsf/P65Kcj5wBXA7cFZVDS7gOv6deB3w3iSX0/xD/rer6qax\n",
       "Fd2jJGcDjwYOTXItsIWmu3qqvjfn4kRvSdIgTVIXpSRJq8aAkyQNkgEnSRokA06SNEgGnCRpkAw4\n",
       "SdIgGXBa15L8oL09zMzjvgvs++1V+Lz3Jfla+1lfaCfOLvUYZyV5UPvz6bNeu3ClNUpqOA9O61qS\n",
       "m6vqoNXed4FjvBf4WFWdk+SxwJuq6pgVHG/FNUmamy04DUqSH0ry123r6ookT5pjn/sk+WzbCrsy\n",
       "yaPa5x+X5HPtez+c5Ifm+5j2zwuAB7bvfXl7rCuT/MZILf+vvenmlUme0T6/PcnPJPkD4MC2jg+0\n",
       "r327/XNbkpNGan5fkqcm2S/JG5Nc0t688sWrc+ak4ZmYpbqkZTowyWXtz18Dngk8papuTnIo8Hn2\n",
       "Xafw2cD5VfW6JPsBd2/3fRVwYlXdkuYu0S8HXrvAZ/8icEWSY4Hn09ynbD/g4iR/AzwA2F1VTwRI\n",
       "MnN/sgKqql6Z5KVVNXqH5ZkulW3t73Jeu97izwMvAX6FZi3B45LcFfjbJH9VVTs7ni9pahhwWu9u\n",
       "GQ2IJAcAr0/yczTrMf5YkntV1fUj77kEeE+7719U1eVJNgEPAT7Xru15F+Bzc3xegDcmeTVwPc29\n",
       "+B4LnNOu4k+Sc2jusnw+8Ka2pfbxqvrbJfxe5wN/2IbbE4C/qarvJ3kc8NNJnt7udzBNK3LnEo4t\n",
       "TQUDTkPzHJpb5hxbVT9ob5Nyt9EdquqCNgB/AXhfkjfT3BH5k1X17EWOX8ArquqcmSeSPIY7328r\n",
       "zcfU1UkeBjwR+L0kn6qqhVqEozV+L8l24PE0LbmzR14+tao+2eU40jTzGpyG5mDg+jbcTgB+fPYO\n",
       "7UjLb1bVu4B3AQ8DLgIemebmmDPXz46a5zNm3zzyAuDJSQ5sr9s9GbggyX2A71XVB4E3tZ8z261J\n",
       "5vuH5p/S3Mx2pjUIzQr6vzbzniQ/0d4aR9IstuC03s0eBvxB4GNJrqC5b9iOOfY9AXhFkluBm4Ff\n",
       "qqobkjwfOLu9tgXNNbm5bhZ6p8+sqsuSvI+m6xOa29Rc3nYnvjHJ7cCtwH+b41jvpLmO94Wqet6s\n",
       "Y/8V8AGabtTb2ufeBRwJfDFNX+r1wFPmOK409ZwmIEkaJLsoJUmDZMBJkgbJgJMkDZIBJ0kaJANO\n",
       "kjRIBpwkaZAMOEnSIBlwkqRB+v94gJlOOuXkJwAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112581650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_curve('log reg, ROC', Y_test, predict, label='l2')\n",
    "roc_curve('log reg, ROC', Y_test, predict_l1, label='l1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def report_cross_validation(grid_searcher, X_test, Y_test):\n",
    "    print 'train score is {:.5f}'.format(grid_searcher.best_score_)\n",
    "    print 'test score is {:.5f}'.format(grid_searcher.score(X_test, Y_test))\n",
    "    print 'best parameter set:'\n",
    "    for k in sorted(grid_searcher.best_params_):\n",
    "        print ' * {}={}'.format(k, grid_searcher.best_params_[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train score is 0.90920\n",
      "test score is 0.86333\n",
      "best parameter set:\n",
      " * C=2048.0\n",
      " * penalty=l1\n"
     ]
    }
   ],
   "source": [
    "c = [2**(x/10.0) for x in xrange(-100, 150, 50)]\n",
    "\n",
    "tree_grid_searcher = GridSearchCV(\n",
    "    estimator=LogisticRegression(), \n",
    "    param_grid={\n",
    "        'C': c,\n",
    "        'penalty': ['l1', 'l2']\n",
    "    }, \n",
    "    scoring='roc_auc', cv=5, n_jobs=4).fit(X_train, Y_train)\n",
    "report_cross_validation(tree_grid_searcher, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "another method max auc = 0.896641057525 with c =  0.03125 2^ -5.0\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEUCAYAAADXzmpaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzsvXl0Hcd95/v9VXffFcDFDhAguO+kSIq0JVmSZcpaIAmS\n",
       "5SWxLCeOE9mJ5pzxnCTnzcSZeQ8RHbyZ57w4mcmJ82YUx4k9iSLJiiVrgSRIskxrl0iR4r6vIAAS\n",
       "JPblLt1dv/dH9SUvLi+Ai+VirY9Oq7urq7rrgvfWt+tXv/oVMTM0Go1Go8kGMd0V0Gg0Gs3sQYuG\n",
       "RqPRaLJGi4ZGo9FoskaLhkaj0WiyRouGRqPRaLJGi4ZGo9FosianokFE9xDRESI6TkTfzXC9iIie\n",
       "I6K9RPQhEa1PuXaGiPYR0R4i+iiX9dRoNBpNdlCu5mkQkQHgKIA7AbQA2AngYWY+nJLnLwH0MnMD\n",
       "Ea0G8HfMfKd37TSArczcmZMKajQajWbM5LKncQOAE8x8hpltAE8BeDAtz1oAvwIAZj4KYAkRlaVc\n",
       "pxzWT6PRaDRjJJeiUQ2gOeX8vJeWyl4AXwYAIroBwGIAC71rDOANItpFRL+fw3pqNBqNJkvMHN47\n",
       "G7vX9wH8DRHtAbAfwB4ArnftVmZu9XoerxPREWZ+O0d11Wg0Gk0W5FI0WgDUpJzXQPU2rsDMfQAe\n",
       "SZ574xinvGut3v4SET0HZe4aIhpEpANnaTQazThg5nGZ/3MpGrsArCSiJQBaATwE4OHUDEQUARBl\n",
       "5oRngvo1M/cTUQiAwcx9RBQGcDeA72V6yHg/+Hggoh3MvE0/Tz9vvj9vLn+2efK8cb9w50w0mNkh\n",
       "ou8AaAJgAPgxMx8moke9648DWAfgJ94HOADgW17xCgDPEVGyjk8w82u5qusYOKOfp5+nnzflz9LP\n",
       "m0HksqcBZn4FwCtpaY+nHL8PYHWGcqcBbM5l3cbJGf08/Tz9vCl/ln7eDELPCB8bO/Tz9PP086b8\n",
       "Wfp5M4icTe6bCoiIp3JMQ6PRaOYCE2k7dU9Do9FoNFmjRUOj0Wg0WaNFQ6PRaDRZo0VDo9FoNFmj\n",
       "RUOj0Wg0WZPTeRoajUYz16ltaMyHmpBcBSAPKoRSB4COpvq66HTWLRdol1uNRqPJktqGRgNAGZRI\n",
       "LPe2Au+yBOAA8EEFbCUAfQDOQU3euwwlJt1N9XXT2vBOpO3UojHDIGy3oL6EEQB+qBAsyU0AsLzN\n",
       "zHBseluq2ZFxbcTh4dJGypPpi5LMk/yxuN7mpGxuyl6OsHHafqT05L1tADZj++z9EmtyRm1DowUg\n",
       "gJTvZFN9nRzjPfIBlEMt67AKKvCqgBKEKIBeAPERbuGD6n0EcfV36UD1Rs5CxeXrANDZVF+XGEvd\n",
       "xkNtQ6MAUPDan93fpUVjFkHYHoQShuRW4W3lAPKRuYFOfs70BjTT8ehVGF+e4coRrv6QUrf0NGB0\n",
       "sUq/NlJdk/e1ASQAxKB+wHHvOJZyHPXyuSmbHGafnjbS33qIyGkBm1pqGxoJqlEuhHrRKoMyEy3w\n",
       "zlO/QwT1b5r8viS3eNpxHEpslnv3ANS/bx+AfmT3GxsJA0DIq7eZUsdeKBE5D+ASgC6oXsmYTVze\n",
       "3yUfQBGAYijRq4FqZ4Kv/dn9f6JFYwZC2O6H+gJXAKj09mVQX0h5JdvQRi7nbxtzEMLQHln6JlL2\n",
       "yfzJL/5wPSjKcJz+zPTzZN50AUo2VE7aefreSTl3MpSTuFbMUo+dDNuVt2zG9ok2dtNGbUOjH6oB\n",
       "L/S2aqjfVDnUvy1w9W8/CPWSkKkHkHyZSX4fUo9Tvy8So/ciJhsflJiEvTqwtx8A0Aa1qF07lJh0\n",
       "NdXXDQBAbUNjGEoYiqDam4VQf59UQYpD/V0GASx87c/u/z+1aMwACNsJ6ktcA2ADgKW42rjEob7I\n",
       "MagfsWZuk9rTGu8+/RgY3WQ4XF3S37JT37aTwuRgqHgl0xIYKkJJIeqH9/bN2G5nWZesqG1o9EE1\n",
       "gIuhfkuVGPp5k41gFBN/85/pWFBiEsK1QsBQL6HJNBtKZAZxdUG7dBZNRDS099QEIWwPQ4nEagDr\n",
       "oWyXgPoxtSL7H7ZmbsEY/kc7XSRFKLlZUONmyXPKcJy+B4b20kjdePsAlEnlMtTbcA+umnP6GNtH\n",
       "fFHyBpgrod6S1wNY4t1bevdqHrbw3MeG+hv0pKUn2+8pfQnVojFGCNtNKHvpYgCboL7oBNWD6Ib6\n",
       "0Wg0M5GkkOVCzCwos1EllBClmgCJsL0f6rfRBSAOiXj+xVCooC0Uyb8QXhRYZVULVxBJSDD1AegV\n",
       "klzhkBSOALkUIgZIEhMTg8HeMejKMRiSQJIkMaByTq4hglPWLkrePOMj0h7MNMzYnOAr15kAFiyY\n",
       "GCyYQCAp1BUW3sdR6SqfyPBkHrq4UqalloyEER7lY46IFo0s8MxOKwBshepRmNBvQNMCuSA2dO9t\n",
       "BmJ7W98w1y0zalT4+s0N/gFfhTVolhHDZwdc6ljebbsmxwHIFBtcshfDxFANpdc+M0CpA0gpX4ah\n",
       "jejVixP5vlzb3KfdjeMGOZ8syHc+ropwzDR8t5+6ZGy60E9iaEkaph6cYnpM/1ykPjSDQUTglDR4\n",
       "f5cUFbu2pmmyQsQAuVSczQcfjpyKBhHdA+B/QA0u/QMz/0Xa9SIA/whgGdSb+iPMfDCbslMBYbsB\n",
       "5WZ3J9QbVD9U13ummR1mLwwYtjCNhPAZtmEJW/gMh0zhCJ+ZMIJGQgSF420u+UmSxcSO65P9rs/t\n",
       "tYNOr+N3Bx2/G7MDbswJODEtKjMDckGBHn8k2O0vDfb4aoyEUQAALDghTXmJBavfkS1g5agOPAlf\n",
       "hVTVYJdgH6goiO+qLrEPlZc6J0tKnJb8IlEc7beWdnXAlLL/yY2fYinIt/5im/+m5tbAXccvmAv6\n",
       "p3JAfUQcn4yMnmt4cjYQTkQGgKNQDW4LgJ0AHmbmwyl5/hJALzM3ENFqAH/HzHdmU9Yrn5OBcM8E\n",
       "tQbAXQBKAXRi+DcoTbYwEOj15Qd6fMXBbn+VkRAFwhX+YbMTuyC4LNhlYpcFXBBLMAmSZAqXLJLk\n",
       "Sy8nTTno+N0+x+/22QGnz/HLmBNwoo7fTbg+mcjK4VgzLsyo4Q92+4tCXf4qX7+1gJgMgNm1eJAN\n",
       "Oas8A1kCzvGScHznwhL7YEWpfaKkxDkXKRZhO24u7eqwVl3u8G1q6/DdcL7TKI7aqeXsQ+X5sV8u\n",
       "XxD/uLrKPlpaYZQP9Pk2XWgN3HqmLbDt1CUKuNM2eO/45JL2dX9064zzniKizwB4jJnv8c7/FACY\n",
       "+fspeV4C8H1mfsc7PwHgZij/6BHLeumTKhrexLr1UGJRCDXppn+y7j8fMeLCCnb7i4I9/nJ/r1Wd\n",
       "FAlpyBgbnLjytjmJkCSTlKBYqtEaijRkXJocdS05KC036vjkoOtz467JtmspYXH9bkKak1+3OYcE\n",
       "BXp8BcEef0mw219jxo0IALBg2zXlIARPS+PIEoh/sKg4/t6iCraFgCsIkohdISCJ4ApilwguCUjv\n",
       "WKULSCLZ6w84Z4tKIKQ0l3QrgbjuQofvxuaOsfYaOG5Q7K0lZbG3lyxIfLKgyr2YX2CtvNzu29ra\n",
       "Gvz8yTbrugu9NIVRACcqGrk0T1VjqL3/PIAb0/LsBfBlAO8Q0Q1Qg8sLsyw7aXjzKTZC9WzCUAN2\n",
       "Z3P1vDlNsjfR7SsJdvsXWjGzGFC9BmnJAcdyBnNeBcEOC3ag3DGvgSQZJMmyokYJDZhmJmEBACaW\n",
       "0uSoNGRMmhyXpoy7loxJS8alIW1psuOa0pamdFxL7aU194SGXBLCIdNwhCEcYQqHTMM2/MFuX4W/\n",
       "z1dFkizA6+EFnK7prKvTmu8f+NdNy2K/XL6cY6blu77tPPkdFwZLCMlkMMNyJQyWwpAMwUymlBDM\n",
       "MCSTwRKGZBGJJXyfbukwF3dFJ9qgk9/l4F0n24N3nWwHsNe9kOeLvrG8Mv7BoqrO59euBwBrXXub\n",
       "b3Nbu//WMxetNZf6cyki8nJoQu1+LkUjmy7M9wH8DRHtAbAfwB6o8YIpsUl7M7M3A/g8lK9zMjaM\n",
       "Zgxc6U10+SsDqhHxAao34fjdrowuHNOI6t2wy8YoEymZBEkYhiNCho08YtMghgEm7w2Nk2OyqYVY\n",
       "Gpxgg+PS4ITaZMITnDgb7EiDHWnK1L3rCY8DkbvvPrkgI2H4zIThM2zhU+NHwhK28AmH/MIVPuGS\n",
       "T7jkI3VskYR17Wf0PqlgR5pykAVPa2+c4wYNvrC2evClNcudo6WVvs1tzQV/+N7OwF0nLpIxs757\n",
       "AGBU9ifyfnvvubzf3nuOJWDvryyIvbWkMv7RwgX9/7J5MwBYqy5f9G1qa/fffO6ib3Nbz3g/h9sR\n",
       "tOIfLSy29y0osY+VljhnikrkoBUE/mDc9c+laLRAzV9IUgPVY7gCM/cBeCR5TkSnAZyEmuswYtmU\n",
       "MttTTncw847RKubNrdgC4HYoV8F2KB/z2YHrEgYHLQwM+BCN+hCLqS0e9yGRsMBMME0XpukM2VuW\n",
       "C8tSxz6fOvf7Hfh8LgxDQozt9SbQ48svbM5b4xuwFgBXexMseCAnn3uqIZZsQDI4+4lrygXUAJNh\n",
       "2BQ2EsgnNgUYBvGo74+kxm44wQbbLNiWyoRnS1MmXJMT0pQJFuxKg1022JGCXWlIlw12DVtYhi18\n",
       "whGWkTCCZkIEhS1ChudI4PUIMrQ+zEyQILhMLEGQTCxdk6MgDMw00U8S/7gqMvCz65bH3128zCgb\n",
       "6AvWHj8R+u+N76WOL8x0SAC+TRd6fZsu9AI4xhKwj5Tlxd9dXJ7YU1Ux+It1azlq+cyVHe2+6y60\n",
       "+29qvui/obmLfPKafxS3K2Amdi4sTuyrLLGPlpU4p4tKZK8/aFb3dlHkuSi4iXwbu1uoKh6MPoFF\n",
       "465zDsc0TKjB7DugJrl9hGsHwiMAosycIKLfB3ALM/9uNmW98mMa0/BcZz8N4D4or6yLmM6wHX19\n",
       "Ply4EEFHRwQ9PRH09xdgYCCCaDSCeDwC2w5BSgtS+uC6PkipNmYDQtgQInFlM4yEl2aDSILZgJQm\n",
       "pDTBbFzZUs+vHquXByIJItXTy3RMpOIskWBBlings0AkmcgmYUgShgthSCLDhTBcEqZM7tU1U5Iw\n",
       "vXPThTAkSDCRYFVOMEioNGEwyEsTgokMCWEwkSHJtCSZQUdYAYfMgCN8IYeEOTNbtrHCJIhBYAhi\n",
       "Elf2gBjayxn1RswCjidCzhUnglmOeynkG3hq45Lo6yuXy55AMPDZM6fCD+076bvu4px1VLFPFwXj\n",
       "by8pT+yuqrCPlJa7XcGwtazrkrXhYjsFbcc5Vlpsny4qkZ3BPLO6r8tc1tlhrW3v8G1p7fBtausl\n",
       "c2gjP2MHwgGAiO7FVbfZHzPz/0NEjwIAMz/uDZb/BOrt5wCAbzFzz3BlM9w/a9EgbM8D8CBUSIIW\n",
       "TIVYdHQEsXfvSrS3L0A0elUMEokIEokCMBvw+Xrg8/XA7+9FINCDUKgb4XAv8vN7EA4PIhBIIBBI\n",
       "IBhU+3A4gUDAGWuvYFQch2DbBlxXwHEIjqOOXZfgugZcV1ACVl5H3qJQX2gFpIRjJAaYHYJ0BLu2\n",
       "YGkb7DrJvQFpC5aOwdIR6tw7lo4B6QpmKcCSwFKw2hOYhxyDJal8TFeO1X1Mdh1T3csxQcQkTAfC\n",
       "dMgwHRKWQ4blkDBdMiwHyWNhSNAVQZNX0q6KmSTD9ATOcr1zCWFKMiyXhKX2hiW9zSXDJ8mwJIQl\n",
       "Kds2XZM1nBAUfWVV5eCLa1fYByqqrPXtraEHDp8M3n+kLb1BnDaYSLiknDxMGcvlo9wLeb7YW0vK\n",
       "4x9XVyBuGtba9g7f5rZO3+a2bvK7o/49ZrRo5JpsRYOwfRmAh6ACgrXlrEKXLoWwd+9KNDevRkfH\n",
       "SkSjFcjPP43CwnMIh3uQl9eDgoJeFBf3oKKiB4WF0Ulv/HMAuaCCtnB1QVv4OpJkuT63NxdeT+OF\n",
       "mcFuXLAdNaUdM9mOmdKJmuzETXbipvT2LB2RIlwC0vWOXZXOrpfmCiVqriduKeksvb0rWEoD7Apm\n",
       "tQczqZ6S4ZIQEiQkqXN1nJomvDQyGCRc1bMS8qqoGUlR83pupvTSrogbPIGjpMBZfkdYQYesoC2s\n",
       "kCOskE1W0J2NQsYxQ0RfXVUZfWPFosSeBTWiJNofvOPEyfDX9p0xKgam13VXkhAu+ZVIeH9cYnb8\n",
       "bjeYhBk3CgGwa7n9bIzBtDlFzGTvqWnHm2/xOSivqEuY7EHuixfzsG/fSjQ3r0JHxyrEYqUoKDiF\n",
       "8vJjuO22J7Fx41kEAjOmcR0zDORdCpZFzoevM2yjwLXcXvZN76BnJogIZAYkzEDCCE6fuZFZgt2E\n",
       "YCdhqJ6XI+AJFLu2EiJpJ0VKXBUsN+X4iqAJZlf10NgR7MZNaTvXiljquWubLG2T3YTFrmOxtE1I\n",
       "KdJ6XzYZvtR9cnNJWA5UL8rbfI63V8emzxWm3yXD7whfyDHCpVGaRDcfOWCJ6Murq2K/XL4osbdy\n",
       "oVHZ3+O/5ey5gv/w3n5rdce0jJORJFM4wp86H4gF23bA6Urk2ZcTIafXDjkD8bA9mHRi8PdZ4fDl\n",
       "QGWoM7DMiBn53lhf/0x60ZoIc7anQdheAuA3ACyCGkSf+D9Ya2s+9u9fhfPnlUgkEkWeSBzF8uXH\n",
       "sHHjOfj9c+KLEezyRwqb89Zbg2a5tORArrvcmtzA0iWZGDClHTXZHrTUPmpJJ2ayHTelE7PYTRjs\n",
       "2iakbbBrG8rsZxssHQOubXomRoNdx1Q9NddgN+FjaZuGv6BfBCK9Rqi4zwyX9Jt5FX1WQVWvWVA5\n",
       "mM04k+z1G4MvrqmO/WrZYvtARZVR09MZuOXsueCDh89ZS7smf6nUFEcF4ivjRMONF5E05aAddDrj\n",
       "efZlO+j0x/PsfifoZjdPg4Fgl78w73KgOtDtX0JMJgtOuJbsn07nAm2eyvDBCduvA/AVqFg4E/eK\n",
       "eu+9ldi5sw59fUsQiZxAefkxrFhxFBs3NsOyZv3gYiq+fjNU2Jy/OtDrW8yC467PnXE9C83MwI33\n",
       "m05vS77deyHf6W/Pdwc78t1od76M9eRLOxoQvrwBI1jYZwSL+oxwSa+ZV95v5pX382ARxd/cVJ54\n",
       "Z22VfWBJhbWk97L/s2fOhb54+JxZ3Ttp4TaMhJFPkswhrtHELAXH+ao7dFxaHHdNGZeWjEvBrjSl\n",
       "7VoykQjbA65fTop5SThkhDoCJeHLgUX+fl8VAJKGjEpLTvka4lo0Uj44YXsAyjPq01BjF+N/O5YS\n",
       "eOedtfj44zrE44VYt+5l3H33h7Pa3DQCRlxYhc35y8MdgVVMLF2f7JmprpaamY+0Y4bd25Ln9F7I\n",
       "c/rbC5yunoh90S6Vsf48SMdEoF/CNwhQXKixHNPxxmsc5VmnHBiunBum63nQJZ0ZpOexJ0GmGi/y\n",
       "xn2IfGTC73cstysa7G323JYdNmDDIBdEEkJICMHe/uoxEXvu5+pYbfDOcSVNiKH7ZD4a4UfjtVXC\n",
       "MaxgX7gk3BteZCZ8RQxm13CiTLYNKYkhCVImHT9IjZsxwCqd2fUcSFSwX1xpw9NEgPlqjMQrtWKS\n",
       "fqui956frp/3YxqE7dVQg90RqEXcx9fiSQns2HEd9uypg+MEsWHDy7j77p1zrUeRir/PCpceL7zJ\n",
       "sEXY8bk90xX6YTQ4IUj2+U1RGLNn4qQtzVWEFXCNxKZY/NWHyxJvLV1gnyoq86291BbYdvpA8AuH\n",
       "W5JzKa46McRN6cQMdmImO3HDc2Aw2E2oczdhXnVASI79eOdu3JS2clIglwLMNgbQ2y05GgbzeuV5\n",
       "x2LIBiTT1DohyWN1LZkn2dCTlz/ZyAqvoU5Nv3o8knCoCCcY8DZVThgEwyCIpCiplRg9MSIQgwQD\n",
       "YOWi7okUhHLOHhpMja/cVu2ujcpuWoEx/nMOYdb3NMCPGVAhRuqglmfsHtfNXJfw5pubsXfvfWA2\n",
       "cN11jbjzzt0w54j//zAEO/2FpScjnwHArs+dksFG2W8ZsTeXl7utBSE5YFk86LN4wLJk1DuOmhbH\n",
       "TJOjlsVx0+KYaXHctOCSgCUdSBJG6UCfsaCv16ju7TGXdPVaKzt6rfUXe42S2TOxay7inC8IDP5i\n",
       "XU3srSWL3XOFJdb69tbA7SfPhh440ioi8ZwtFmQkjDyS5BssiR3tWtR3wvXJWbc6pr/PCgc7/WWh\n",
       "rsDipAeWNGRswiYsJuHN+A8AQKwgHmlf98fz1zwFfuybANZCxaoa+xfFcQivv74V+/ffByEcbN7c\n",
       "iNtv3wdj7r/K5l0Mlhefyb9RGhzPpW2VJWDvW1AQfWN5dWJXdZV9qrjMXNjTaVT291HItilo2yKc\n",
       "sCmccCgvYYu8hC3y4zblx21RELdFJGaLwphNeXGXBCB7/GbiYEWBc7ykwDldVOCcjxS4bfkF7qVw\n",
       "Aflcx6jo7zWqenvMRT295rLOXmv1pV5r1eX+TLNoNRPHOVMYHPzFukWxdxYvclsiRdZ1F1qCnz95\n",
       "Llh3tFXkJ3JqziWXLDNhRBJBp71rSd++WCQxJyb5+QbMYLDTXxrqDCyyYmYJcCXQ5uBoZmOSZAhH\n",
       "BJIeX0wsnaDbEctPXIjnJ3qikUSEre1/Op9F4zEAF8Zc2LYFmppuwMGD98KyBrFly0u47baDs2He\n",
       "xGQQac5bFGkJb3F9si8XIavdjqAVfW1lZfz9RVX2vsoqALA2XmgN3Hy2NXDXiQu56BGwBJwzRSH7\n",
       "cFmBc7KkwDlXWOC2FBS4F/Iiss8fFHmJqCiMRUVhNCqKooOiZDBqlA9EjfL+qFHVO2jU9ESNiv74\n",
       "VEYcna3YJ4pDg8+vXRR/Z8li90JexLfpwvnAnSfOBu891ibCdu7Nm2o+RIQF2z3V/Xt7Fwy2ztWQ\n",
       "92bU8Ic6A6WhTv9C36BVAYBYcNy15CCImVyyhCOCycCbLNhOhO2LsYJEezzf7onnJfrT1phZBPre\n",
       "uNcInwui8eiYCsXjBl599TM4fPge+P1d+PSnG3HzzUfmi1iAgZJTBavzLgXXOX63a7J8x9klxD+o\n",
       "KY7tWFaV2F1V5TRHiq3lne2+rS1twTtPtlgb26Y0/HM6csASbnMk6LQWhNy2/KC8FA66l8Ih2REK\n",
       "yu5AUHYHQ7LXH+S4aYr8eJQisahRFI2KomhUlAwOGhX9g0aFEhezpicqygYSc11cWALuucJg4kBF\n",
       "xDlRErFPF0XcloKIeyE/Aknku771fODOE2dD9x67MJXrQxgJI0ySAtGi2PHOxX3HXf/sWqdjIhhx\n",
       "YYU6AyWhzsBC/4C5AEzkWm5/Is++GMu3O+L5id5E2BkcRUC1aGRd4OjRcrz44rdhmjHcdNMLuOmm\n",
       "Ezms3kQwsthSm6zh/hGTK0cCAMGFiLSGVwR6fdWOT3YT2FWrK4OJSYLVf8RgwFuHGSqF1DAcI225\n",
       "48Hn1ywYfHn1MvtgxQIK2gnfprbWwK1nWwN3nLyYa9NELpADlnDPFwSdlkjIbcsPuu15QXk5FJId\n",
       "oaDbGQzJnkCQewIhdoQQkXhURKKDoiimhKV0IGqUDwwalX1Ro7p3UJQOJIziaCKb0A7TCbsE53hJ\n",
       "OHGovNA5URJxzhRF3NaCiNOWFyFTSmNBX7dR3dtjLe3qMVd09Pg2XOwxFnVPOGT4WCGXLCNhFDgB\n",
       "t6Nzce++WFGiZ2prMLMQtjBJQoxDNCckGnPGe2pEpARefPFWHDjwJaxb9wIefPDXU9SzMADkQ4Vd\n",
       "Tzb2nLJdM5nI28eh3IXjAAa94+QW9bYYrq7LnL4B6t/W8vZmsNMfWvbOgvvMuNEfz7PfcS1pSlNa\n",
       "riVNaUhTmmxJgwULNkAsWEBItci94S12n1zKXgAg2Rk0B/707nXOgYriwMP7TgYf+9W7xsqOGEit\n",
       "6Wyrz62WOvZURvmbMHlyJNXpFcFjz2OQr033Cl65CqkkjpgYkpTMyWsXcx47ImxLsbpjYLQZyLLX\n",
       "bzjnIiG3pSDoXsgPue15QfdyKGQfKy2RXcGg7A4EOWr5OWZaMKVLQSchgrZNQTtB4USCQnZChBMJ\n",
       "ykskRF7Cpvx4QhTEEqLAG8tR4zuOKIg7Ij/uUF7CGYvHGEtAdgUt90J+wG0P++XlcEB2hPxuVzAg\n",
       "uwMB7gn4Za8/ILuDQfdSqECE7bhR2ddj1PR0W+svXg59+eAJ34aLPUZl//S/xSdNUcRuz8L+XT1V\n",
       "Ay25DCE/W5DW9Az2z33RaG8P45lnvoHBwTJ86Us/wLp1uYs9BYQAFABXljy2odx/26Aa/6iXlvA2\n",
       "O9M5Y/uk/iBqGxoLAPw2AD+ADyd6v1NvL1tz+q0V3wxX9B1c/+g7f20VRx33ZJ7gM2EhTSmkwUKa\n",
       "0mDBJE02pCEFCxbSYMEGC9eQhrSkKQULEIQUTCAmEJQ8EQQT48o5IEAACzaYmFhAsCFNabDpmuxj\n",
       "IU11DYLUTC5PrZSAMV2RI/IucLK75J1CTf5ikNfDArwelydM6jyZAoniKJtF0ai58eIAJXtoGUSL\n",
       "JcB9ftPtCPlkZ9CS3UGf7An4ZE/Ax71+n+z3+dzOYICbIwU84PPJAcvHcdNE3DA5blqcMExOGCYc\n",
       "w4DpuuRzHfK7NvkcB37XIb/jUMBxyJSuHPD5uM8fkP2+AA/4/DClK/ISccqLx0R+PC4K4jFRGIsZ\n",
       "JYNRsaKjS5QMxoyK/pi1rr3XKIrNOG8jYYugcEUQxDJaFDvZtaj/uBNwp1/E5jlzWzTefnsN3n77\n",
       "91BVtROPPPIPCAYn84dhQb1Nh7xzggq1vhNq1b+LALoY26d1zkNtQ2MZgG9CrVHSMpF7xXr9vr1P\n",
       "b/1K/8X8TUtuPfm/l287cQgA0HfNMt3TghSSpMFCWlK4phSqD8VX9iyYpFDCxcTk9ayG9KOkIQ1p\n",
       "suHlM6XBQhrS8I5NabCSPkOaUsBkQ0mhFGzAU57ULiQDjHybqLqfGYBgOAJwQJ6bfsrcK0KK7Y+Q\n",
       "vJO6pSvAfT5Cr9/gPr/gPr/BfT6DB3wG9/sNxAwSkVjcKBmMGWUDg0Z5f9QIOzNyvs1IkCTDSIh8\n",
       "gITjdzt7F/QfHCiLXpyNLrRzlbkpGtGoiaeffhCtrTfg1lt/gttuOzx6oVHxAyjF1bGEKIBTAE5A\n",
       "CUQ7Y/tG1/1FAAAgAElEQVSkhUCYDGobGhdBCYYNVcdxc+6DxctOvLn694KFg6du+nfv/Hm4dCDn\n",
       "y7aOFSEFCwkXtjEtYymuIYm9npZryhRBAnnCpGZqJY+VuY+8/6t86WJmsJBKzEw2WEiDTRkZFFw8\n",
       "YHgCJqRg05MxyzXZJ01p2QZHHFzVpGRPiwEmumr9I29MC0ySZIqpT6UnUybF9DcsTDBsESZJARYc\n",
       "HyiJH+8vHzwfL7B1CJsZyNwTjUOHKtHY+G0Egx341rcaUFEx0S9eAEA5lEi8BTUf5CKA3sk2I00m\n",
       "tQ2NawD8FtRkx3H7ricGLXPv01u+0HO+8KZFN5x9clXtkT2TVsk5huEKhitcTEZwzAnCxHBNaTh+\n",
       "13B9run6pOH6pOlarula0nAtabqWNBy/6/fGtixpSsM12ZKmtFTPSlrSJ62k6Q9Xx5vAauwKTIBw\n",
       "yTFsSghH2DSG2DPkkmXYRh4AJEL2xf7y6OmB0thlNmZmRAKNYu6IhpTA889/DocOfQEbNjyHBx54\n",
       "Z4KD3UEosRgA8CKAT2ZaT2I4ahsabwTwRaj5K+OetNeyp7rm2KvrHvGFExdv+Pb7DQULeufExKn5\n",
       "ADHBtA3XVL2uCY8DuIYkaUnDNaVwLWm4fteMFSSCsfxEOFoYL4zn2wWJsBMBqcAWTAAxpOGIhLAp\n",
       "IaRQQsokjITIIyZTmnKwr2LwQH9ZtM0OO1MeuE8zPnIqGkR0D66uvvcPzPwXaddLAfwLgEqvLj9g\n",
       "5p94185AhQVxAdjMfMOwD7pwIQ/PPPNNxOOF+MpX/l+sWTMRU0wQQBmUWDwHYB9j+6wZfKttaLwJ\n",
       "SjCacdWTakw4ccPY98z193SeKr29+vrmZ9bUHfxwrs9J0IyM4Qo2XOFYKWkFF8JDXiKYGNFIPBiL\n",
       "JILxfDsYjcQLYpFEJB62Iy45BeQKP8A8UOxcjBXGz8Tz7D7P2FsMQEL91mXKsZshXWK8ceU0k0LO\n",
       "RIOIDAA/hFoAqQXATiJ6IW2d7+8A2MPM/9kTkKNE9C/MnDTHbmPmzhEf9Otfr8e77/4OFi78AF/9\n",
       "6v+aQBTaEJRY9OKqWMyqOEa1DY2lUDG4zmE8IVUAXDhYueBI4/rfMyw5sPWbH/7fRYu7xhfLSzPv\n",
       "ICaEugPRUHcg2WvoBFACoNUOOG2Xl/UcPHPzhUt9CwZdeK7g3ubPsPm8LfU86UJupD06k4iM5NKe\n",
       "epzqAi/T9shwnF4mfZsM0uuZqd7ZXBsuLYQJkMuexg0ATjDzGQAgoqeg1uhOFY02ABu94wIAHZ5g\n",
       "JBl99O29976Bz33uH3HLLUfHWc8w1Be7F8AzAA7ONrEAgNqGRgJwP9TcjjELhnSJ9v/b9XdcOlp+\n",
       "b8X6tuc3fGnvWzOkd6GikF49JoyzB6WZEkwohxEf1HhaE4Ajb/6nBydt1UzC9uR3IikgRtpxeprw\n",
       "ttTj1DQzbUumJa9TSjqlpKffO3l9fB/rqihlEq1MwpbEHaFc8jhV8I4A+ONx1jOnolENZSJJch4q\n",
       "Gm0qPwLwJhG1QrmvfjXlGgN4g4hcAI8z848yPuXb3/5zlJWNx5MnD0osugA8DeAwY/tsduu7DsBq\n",
       "AKfHWtCJmcYHj9/yqGsb4c0Pf/z90pWXJr5w1fDkQ4WvT2WkN8JUE0XyOJySj6HmwCQnQWqmHgJQ\n",
       "BPWbsgF8DGAvgPNN9XWTbkryHFBmhMPBbIXwvXGXzaVoZPNl+S8APmHmbUS0HMDrRLSJmfsA3MLM\n",
       "bURU5qUfYea3r7nD77x2x5Xjzy89iv9087FRnpkc4O4A8CSUWMzqL19tQ2MeVC9uzIEbXUfQh/9w\n",
       "8yMA6Jb/8Ou/Nv3uZP8t/FANStIc3gzgPSgTWj+G2qqHbMM1OLUNjaZ3zyKot9oaqJeUGqRMeYAa\n",
       "lxqE6n1pJp88qH8DADgGNUfpVFN93awZA5wvENE2ANsm4165FI0WqB9xkhqo3kYqNwP4rwDAzCeJ\n",
       "6DTU2/IuZm7z0i8R0XNQ5q5rReOV33pxDHWqhGpMngJwaLaLRQp3Q3WNx+SBwhLY+Q+f+W03YeTd\n",
       "9O/e+dtJEgwBoBBXewN9AHYBOA715jlhL5mm+joHahnfS1CNFQCgtqHRwlUxKcNVMSnH1ZeYpBkg\n",
       "ASUmce9YD66OjoDqJebh6mTWFwAcbaqv653OimlGhpl3ANiRPCeix8Z7r1yKxi4AK4loCYBWqFX1\n",
       "Hk7LcwRqoPxdIqqAEoxTRBQCYDBzHxGFoRrF8fenlH21GsAhAC8wts+ZQGe1DY3LAHwKahZ61rAE\n",
       "dv7TZ74a7wtU3fgH7/4PX8ieiGkuDCUUyQb5BIADUC8JHbkwUWSiqb7OBtDubVfGuGobGv1e/fK8\n",
       "uhZAmSaTAlOCq4OrqeKSDPGSjPs13+YPENTfLOIduwBOQv2Ompvq69qnsW6aaSJnosHMDhF9B2og\n",
       "zADwY2Y+TESPetcfB/DfAPwTEe2Feov5E2buJKJlAJ4ltRa8CeAJZn5tnFUphTKRPAtg10yekDdW\n",
       "vMbwy1CmtjF9rt1PfPqBgUvh1Td8+/2/CkRi4zXf+AFUQTk0/ApqPKXVa7xnDE31dXGot+KMrtie\n",
       "E4EfSlCSWwjKFbQE6jtUAfVdTBWVpKBEMQlzIWYIISiBTYroOQDvQ5kV27xenmYeM5dDoxsAFkK9\n",
       "7f6csX3OvRXVNjTeAWWnbB4l6xA+eWrLXZ2nSj77qd/98C8LqsY9YS8C9cb+TFN93b5x3mPW4AlL\n",
       "CGogPx/qs5dDiUkprg7up46p2FCebA7UW7qTss0ECKoXXgglmgxl8juIqy8A2rlgDkJErNfTGEoE\n",
       "6ofwBoC3Z7lXVEZqGxoroea5tGAMXiQHnt342YuHK+/d8lu7/rJoSWfXOB9fAdUg/nNTfd2EgiDO\n",
       "FWobGpNh8JOCUugdJ3stISgnjOQeyNw7TPWvT7pWJr3Hkl5DMu04eW4gJRy+d5wM/5H6rOQzJNSY\n",
       "0xEok2JLU32dnvU/D9CikZIENXbRC+BnjO3npqdmucVroL4NZT7J2v/9cOO6T7furvmNjQ/t/kHZ\n",
       "qnG51RLU4PJZAE/rwc/x4fVakpPaUiexJY9TG30rJT25WRn2SUeIASivtH4oQUh6jyW35Dot8Zlm\n",
       "RtRMHRMRjbkTe0q9wVUA+ABAE2P7XO5WXw9gEcYw+H38jVUbW3Yv+uqGL+797+MUDAvK3Pc+gFd0\n",
       "gzN+PMeA1AWzNJpZw1wRjQVQXfWfMLaPd2b4rKC2oTECNfM768WkTr+9bPXZ95b9zpr7Dv5t5XVt\n",
       "reN4bB6U3f45ADunyhtKo9HMPOaCaCyFGrh7gbF9TptLPLPGfbg6z2BUmj9atPTkr1b9wYrPH3t8\n",
       "4aeax+SW61EKZfr4UVN93Zhnm2s0mrnFXBCNfwOwey650o7AKqhYXVk13q17qxcefXXdv19y68l/\n",
       "WnLrqdFmymdiIZSb6pNN9XUjB47UaDTzglk/ED7ewZzZRm1DYxDAH0INYg6Mlr/9cEXF/n/b/H8s\n",
       "/NS5p1ffe/jjMT7OhBKMTwA8781z0Gg0cwQ9ED4/uB3KVXPUN/7OUyXF+3+++Y8qr2v9xTgEIwgV\n",
       "buVlAO/o8QuNRpOKFo1ZQG1DYw2AW5DFJL6elkjBJ09t/ePSVe2vr//i/vfG+KhiKNfPf2qqrxuP\n",
       "OUuj0cxxtGjMcLyIrl8E0INRYh+xBPY+veWRSHX3zk1f3fPmGB9VAeXb/486ppBGoxmOmbHMjmYk\n",
       "boBq0EddQe/Ya2u3uAmzYPPXd700xmcUQ42T/L0WDI1GMxJaNGYwtQ2NJQDuQRZzMmK9ft/5XYt+\n",
       "c+lnTzxpWHIs0ViDAAIAnmiqrxt1gF2j0cxvtGjMULw5GQ9AzccYdebwgec23RcsGjyx5JbTx8fw\n",
       "GANq0PvJpvq6jBFgNRqNJhUtGjOXjVDri4xqLrp0tLy8+2zxZ9d9Yf+/jfEZNQBebaqvm9Oz6DUa\n",
       "zeShRWMGUtvQWAq1TsaoIT9YAkdeXve10pXtTYU13WNZXKoaah3na1dD1Gg0mmHQojHD8JYs/Squ\n",
       "RiUdkRO/XL3JjlklG76075djeEwp1LoJzzfV18231eg0Gs0EyKloENE9RHSEiI4T0XczXC8loleJ\n",
       "6BMiOkBEv5tt2TnM7VCr4V0eLWNiwGed+2jxQ0tuOfWkGXCyXVMjD2os41/1AjsajWas5Ew0iMgA\n",
       "8EMo7591AB4morVp2b4DYA8zb4Zage6viMjMsuyco7ahcQWUaJzPJv/+n2+6J1AQO7PstpNHsnyE\n",
       "BdXLeELHktJoNOMhlz2NGwCcYOYzzGwDeArAg2l52qBWOYO372BmJ8uyc4rahsYCAA9BDXyPajK6\n",
       "fLy0tOtMye3rHjjwTJaPIKh4Us831dedGn9NNRrNfCaXolGNoWEvzntpqfwIwHoiaoUalP3DMZSd\n",
       "M9Q2NAoAX4IyG2U1V+LIy+sfKllx6bUxLNm6EMCHAD4aXy01Go0mt2FEsgl0918AfMLM24hoOYDX\n",
       "iWjTWB5CRNtTTncw846xlJ8h3ATlXnsmm8wnfrnqusSAv/LGP3j38SzvXw4lwo06AKFGM/8gom1Q\n",
       "QwATJpei0QI1DyBJDa611d8M4L8CADOfJKLTUI3n+SzKwiu3fZLqOy3UNjQuhFpYKatxjMSgZZ77\n",
       "YMlDi28+/aQVdJwsihRArWr4tF6iVaOZn3gv0zuS50T02HjvlUvz1C4AK4loCRH5oOz1L6TlOQLg\n",
       "TgAgogoowTiVZdlZj7dGxtegghFmIwA48Oymu3358Zbltx8/mEV2H4BCAP/cVF83ljkcGo1Gk5Gc\n",
       "iYY3oP0dAE0ADgF4mpkPE9GjRPSol+2/AfgUEe0F8AaAP2HmzuHK5qqu04EXJqQOQD6UaIxK56mS\n",
       "4s5TpXeurTvwsyyyC6hxoH9rqq/Lqhej0Wg0o5HT0OjM/AqAV9LSHk85vgwVXymrsnOMjQC2Isul\n",
       "WwHg8Evrv1q8tOOXJcs7OrLIXgPg1031dZ+Mt4IajUaTjp4RPg3UNjSWIcswIUlO7lixLt4XWLjh\n",
       "K580ZZF9AYDjAF4fZxU1Go0mI1o0ppixhgkBADtqmmffW/q1mhvP/MwXskcb+yiEctt9pqm+LttZ\n",
       "4hqNRpMVWjSmnjugwpGPGiYkycHnNt3hC9ntK+88tm+UrAQgAhXqXK+NodFoJh0tGlNIbUPjSgCf\n",
       "g3JHzoqus0WFl0+U3b2m7uDTWWSvBLCzqb4u6/trNBrNWNCiMUXUNjRGoFyHLyKLMCFJDr143W8W\n",
       "Le78denKS5dGyWpB/Xv+avy11Gg0mpHRojEF1DY0GlBhQgjAYLblTr+9bHWsJ7B0w5c/eTWL7JUA\n",
       "3tDzMTQaTS7RojE1fAbAKqheRlY4ccM4887yh2s+fe5n/vxEYpTsIajBbx1XSqPR5BQtGjmmtqGx\n",
       "GirE+5gm2B14dvMdZsDpWnnnkWzmWZQBeKmpvi4rbyyNRqMZL1o0cs9WKNfarMKEAMCF/QuqLh8v\n",
       "q117//4naPR/oSIoQTo07hpqNBpNlmjRyCHeWMYmANnM4AYAODHTONy4/lsLNrY8W7rycjZuuREA\n",
       "L+tlWzUazVSgRSO3LIQKGph1L+OTp7Y+YAXtznVf2P9uFtkrAOxtqq87O94KajQazVjQopFb1mAM\n",
       "gnHug8XLupuLbtn4m3v+dxZmKQNKkHSoEI1GM2Vo0cgR3mp81yNL01SsJ+A/8cvVjyy68cwTBVW9\n",
       "fVkUWQAVkFCv9a3RaKYMLRq5YwGAMIDR3GUBAJ88teU3gkWDJ1bdnZW3VABqcD0bE5ZGo9FMGlo0\n",
       "csdKZDnz+8SbKzcMXMrfsOlru7MJFQKosYxXmurrouOunUaj0YyDnIoGEd1DREeI6DgRfTfD9f9I\n",
       "RHu8bT8ROURU6F07Q0T7vGuzatKat8DSVgCjmo76LubnnX132TeWbTv+k1DxYDYiEAHQDmDvBKup\n",
       "0Wg0YyZnizARkQHgh1DLubYA2ElEL6SuwMfMPwDwAy///QD+iJm7k5cBbGPm2WizL4MKUd48UiaW\n",
       "wL6fXf/1/KqeXUtvPXU0y3sXAfixDnuu0Wimg1z2NG4AcIKZzzCzDeApAA+OkP/rAJ5MS6NcVS7H\n",
       "LIcSvRE58vL6GxL9/qrNX9v9iyzvWw7gcFN93ckJ1U6j0WjGSS5FoxpD37TPe2nXQEQhALUAfp6S\n",
       "zADeIKJdRPT7OatlbtgKoHukDF1niotadtd8dVXt4X/0hRN2FvcUUAPg2azcp9FoNDkhl2uEj/qm\n",
       "ncIDAN5JMU0BwC3M3EZEZQBeJ6IjzPz25FZx8qltaCyGijh7brg80iXa/+ym3y1edvnN6i3nh82X\n",
       "xgIA7zXV17VPRj01Go1mPORSNFoA1KSc12D4oH1fQ5ppipnbvP0lInoOytx1jWgQ0faU0x3MvGP8\n",
       "VZ4UlmIUwTzw7KZt7Ar/pod2ZxPyHFCT+FwAb020chqNZv5BRNsAbJuMe+VSNHYBWElESwC0Qi1A\n",
       "9HB6JiKKALgNakwjmRYCYDBzHxGFAdwN4HuZHsLM2ye74hNkK4Bh17S4eKiisv1w5f0bv7r7LwxL\n",
       "ZhsvqhIqim3/pNRQo9HMK7yX6R3JcyJ6bLz3yploMLNDRN+BssEbAH7MzIeJ6FHv+uNe1i8CaGLm\n",
       "VHfTCgDPEVGyjk8w82u5qutkUdvQmA/Vo8roNeXEDePwSxseqVjX9kL5mvZszUx5UOMjH09SNTUa\n",
       "jWbc5LKnAWZ+BcAraWmPp53/FMBP09JOA9icy7rliKUjXdz3sy33Gpbbv+HLe389hnuWAfhpU31d\n",
       "NoPlGo1Gk1OG9Z7yJub9Zob03yCiu3JbrVnL9QAympDO76pZ3Hm6ZNt1X9mbTTDCJCUATgDIdg6H\n",
       "RqPR5JSRmq8/A5DpjfjXABpyU53ZS21DYwhqfsY1rrbxPp/v2GtrHln4qXNPFS7qGtEVNwWCMk29\n",
       "2lRfNxZPNI1Go8kZI4mGn5mvsbsz8yWoQHyaoSyG+nte08DvfXrrlwIFseY19x3aNYb7LQCwq6m+\n",
       "rmWyKqjRaDQTZSTRyCciKz3RSwvkrkqzlk0ABtITz7yzbFXfhYLrNz60+1/HcK8glIvtG5NVOY1G\n",
       "o5kMRhKNZwH8PRHlJROIKB/A4941jUdtQ6MfasGlrvRrLZ8s/FzFurbGvLKBwTHcshLAs031ddms\n",
       "q6HRaDRTxkiiUQ/gIoAzRLSbiHYDOA3gEoD/ayoqN4tYBOVWPGTeRbzP5xvsCK9ffPPpPWO4VyWA\n",
       "AwAOTWL9NBqNZlIY1uXWCzL4p0T05wBWQNnqTzLzWN6Y5wvroBZFGsLZ95et8+fHzuZX9mU7Kc/v\n",
       "7V/Sg98ajWYmMqxoENFXcHVQl6DeoguJ6BNm1mYTj9qGRhNqPONy+rXLx8u2Fi3u3D2G21UBeLqp\n",
       "vm7YGeUajUYznYw0ue8BXOsJVAxgExF9i5l/mbtqzSoWArAAOKmJdtQ0By6HN6z/4r5nsrxPOYDD\n",
       "0IsraTSaGcxI5qnfzZRORIsBPAMVQFADrAVwzWztM+8tW+sLJ1oi1T29WdzDByU8L2qzlEajmcmM\n",
       "eT0NZj4L1cDNe2obGgXULPBrVhe8fKx8a9GirmxNU1VQgnGN95VGo9HMJMYsGkS0BkAsB3WZjVRB\n",
       "zalIpCY6MdMYaM/bVHPjmWxEowzASQBjGfvQaDSaaWGkgfAXMyQXQTWUv52zGs0uVkFNwhvC2feX\n",
       "rrFC9oWixaOGDLGgPKaeb6qvyzZMukaj0UwbIw2E/1XauYQywxRDicZ7uarUbKC2oZEAbEEG09Sl\n",
       "o+VbIjVZmaaqAbzQVF93jeeVRqPRzESGNU8x847kBqAXypuqEcCfQ3n5zHfKARQizVTn2kL0t+dv\n",
       "rvn02dFEowTAWQA7c1Q/jUajmXRGMk+thlpp7yGoWeDPACBm3jY1VZvxrEDaDHAAOPf+0lVmwO4o\n",
       "Wd7RMUJZE0AIwI+b6uuuMW9pNBrNTGWkgfDDUOaXWma+jZn/Fhns9yPhrclxhIiOE9F3M1z/j0S0\n",
       "x9v2E5FDRIXZlJ0BbEWGWFPtRyq2FC7sHm2VvSoATU31ddmu3qfRaDQzgpFE48sAogDeIqL/RUR3\n",
       "QM0MzwoiMgD8EMA9UGE2Hiaital5mPkHzHw9M18P4D8D2MHM3dmUnU5qGxpLoLyehkS1dR1BfRcL\n",
       "rq/+1LmRTFPFAC4A+CCHVdRoNJqcMNKYxi+Y+SEAGwC8DeCPAZQR0f8koruzuPcNAE4w8xkvjtVT\n",
       "AB4cIf/XATw5zrJTzbJMic0fLV5hWE5v2apLl4YpZwDIB/Dzpvo6Z5g8Go1GM2MZdZ4GM/cz8xPM\n",
       "fD+AGgB7APxpFveuBtCccn7eS7sGIgoBqAXw87GWnSa2IsMKfe2HKrdERjZNVQN4o6m+ri1nNdNo\n",
       "NJocMqbJfczcycx/z8yfzyb7GG79AIB3mDnZEM/YUBq1DY0RqHhTQ4I2Speor61gS/WW5uFMU4VQ\n",
       "QQ3fyXEVNRqNJmeMNE9jorRA9UyS1ED1GDLxNVw1TY2pLBFtTzlNugjnkiXIIGrndy1aKkwZrVh3\n",
       "8UKGMgaACID/2VRfd02cKo1Go8klRLQNwLbJuFcuRWMXgJVEtARAK5Tr7sPpmYgoAuA2qDGNMZUF\n",
       "AGbePnlVzootSOtlAMDFAwu2FFQPa5qqArCjqb5uONHUaDSanOG9TO9InhPRY+O915hjT2ULMzsA\n",
       "vgOgCWoVuqeZ+TARPUpEj6Zk/SKAJmaOjlY2V3XNltqGxjDUIPiQ9S5YAr1tka1Vm1symaYKoFxz\n",
       "35qCKmo0Gk1OyWVPA8z8CoBX0tIeTzv/KYCfZlN2BrDE2w8xT7XsrllMQtqVG1pbMpQpBPCvTfV1\n",
       "16zsp9FoNLONnPU05igbAVyz3O2F/VVbCqp6d9O1f00BNWv8VO6rptFoNLlHi0aW1DY0+gGsQdos\n",
       "cJZAb2tk64KNGU1TRQAONdXX6VDyGo1mTqBFI3uWQXlBDYk31baveiEAqtp8/lyGMmEAn0xB3TQa\n",
       "jWZK0KKRBbUNjQaAu5Eh1lTb3uqt+VU9w5mmXABncl0/jUajmSq0aGTHGqhQ6ENcbVkCPS2RLZUb\n",
       "2jK52hYDOKxNUxqNZi6hRWMUahsaTQD3Qs3mHsLFQwsWsBS+hVvPnclQNAgVckWj0WjmDFo0RmcD\n",
       "lNvsQPqF1j0Lt+Yv6N2TwTSVHPs4k+vKaTQazVSiRWMEahsafVCBFDNGre1pKdxSsX5Y09RBPTdD\n",
       "o9HMNbRojMwmqFDm0fQL7YcrKqQt8mo+fTbTHIwAgL25rpxGo9FMNVo0hsGbl3E3gIuZrrfsrtmS\n",
       "V9m3RxicHrzQgPaa0mg0cxQtGsOzFarHkNHE1HO+cEvF2gvDmaYOaNOURqOZi2jRyEBtQ2MIwB0Y\n",
       "ppdx+XhpqRM3i2puOnM8w+UggH25rJ9Go9FMF1o0MnMDAB+ARKaL53ct2pJX0bvHMGUm05QDbZrS\n",
       "aDRzFC0aadQ2NOZDLVYy7JKs3c1FW8rXtGeKNVUCYH9TfV1GsdFoNJrZjhaNa7kJ6u/iZLrYeaqk\n",
       "2IlZZYtuOn0sw+UAtGlKo9HMYbRopOCt//1ZjNDLaN656Pq88r69pt910y4ZAGxo05RGo5nD5FQ0\n",
       "iOgeIjpCRMeJ6LvD5NlGRHuI6AAR7UhJP0NE+7xrH+WynincCjWTO10QrtDdXLSldFV7Jq+ppGlK\n",
       "rwGu0WjmLDlbuY+IDAA/BHAngBYAO4nohdRlW4moEMDfAahl5vNEVJpyCwawjZk7c1XHVGobGkug\n",
       "TFOZVt8DAHSfKyq0B3xVS24+fSTDZW2a0mg0c55c9jRuAHCCmc8wsw3gKQAPpuX5OoCfM/N5AGDm\n",
       "9KCAlMP6pXMblHlJDpfh3IeLN4fL+veZASe9J2J6Zc/msH4ajUYz7eRSNKoBNKecn/fSUlkJoJiI\n",
       "fkVEu4joGynXGMAbXvrv57CeqG1oLIeazJdxXkaS7nPFW0tXXhpuQt9ebZrSaDRznZyZp6Aa/dGw\n",
       "AGyBmkgXAvA+EX3AzMcB3MrMrURUBuB1IjrCzG/nqK63Q838HrbOva0F+fF+f83iW04dynA5AGB/\n",
       "juqm0Wg0M4ZcikYLgJqU8xqo3kYqzQAuM3MUQJSI3oIKEnicmVsBgJkvEdFzUOaua0SDiLannO5g\n",
       "5h1jqWRtQ2MVgI0AMi3XeoXTby+/IVzav98XstNdcU2oSYAjltdoNJrpgoi2Qc0/mzC5FI1dAFYS\n",
       "0RIArQAeAvBwWp7nAfzQGzT3A7gRwF8TUQiAwcx9RBSGChz4vUwPYebtE6znHQAGMUIvI9od8F86\n",
       "VnHPui/s/5sMl0ugTVMajWYG471M70ieE9Fj471XzkSDmR0i+g6AJqg5DD9m5sNE9Kh3/XFmPkJE\n",
       "r0J5HUkAP2LmQ0S0DMCzRJSs4xPM/Npk17G2oXER1FKuIw5gH35pw52h4oEjVZta0ntKgBI7bZrS\n",
       "aDTzglz2NMDMrwB4JS3t8bTzHwD4QVraKQCbc1m32oZGgnIH7h8pX9+F/LzOU6Wf3/zwx9/PcFmb\n",
       "pjQazbxiPs8IXwpgBYCOkTIdblx/b35l767SlZcyrd5XCuCTpvq6jCFHNBqNZq4xL0XD62XUAuge\n",
       "KV/n6eLi3pbCz6ypO9g4TBYftGlKo9HMI+alaEDND6nBKKJx9NV1DxQt7vx1pLqnN8PlpGmqOcM1\n",
       "jUajmZPMO9GobWgUAO4BMGJ4kgsHKxcMXMq7bu0DB4YbgC8FsFubpjQazXxiXomGt+73fQAqAGTq\n",
       "PVzh5C9XP1i6qr0pVDwYHSaLD8DBSa6iRqPRzGjmjWjUNjQuBPDvoSYJjmhSav5o0dJYb2DJugcO\n",
       "7BgmiwU1g1x7TWk0mnlFTl1uZwK1DY0WVMjzuwB04dpZ6UNgCZx+e8WXK9a3veQLJ4absFcC4OOm\n",
       "+rphQ6hrNBrNXGROi0ZtQ2MFgN8AUAXVuxi1kT/11op1TtwoWHv/gfdGyKZNUxqNZl4yJ0WjtqHR\n",
       "gLx/+7cAABcaSURBVDJD3QcVIiSrkOXSJWr+cMmXqrc0/8Kw5HAh0i0AUWivKY1GMw+Zc6JR29BY\n",
       "DOBLAJZBxbzKOibUsaa1W0HMq+4+smeEbKUAPtKmKY1GMx+ZM6LhTdi7HmqhpzEviOTEDaP1k4UP\n",
       "Lr315BM0snuABSBTeHSNRqOZ88wJ0ahtaCwA8ACADVC9i/hY73H4pQ23WAH78tLbTmZayjWJD8o0\n",
       "NeJgukaj0cxVZr1o1DY0roUa7CYAp8dzj3ifz9d+qLJu9X2H/r9RspYA+FCbpjQazXxl1osGgN8B\n",
       "cAGqBzAuDr143ecDhdGTC7c2j2bS0qYpjUYzr5kLk/vOYAKCMXA5HOo4UXbXyjuPPj9K1qRpqmW8\n",
       "z9JoNJrZzlwQjWzWIh+Wwy9uqM2r6NtTvvbixVGylgHYqU1TGo1mPpNT0SCie4joCBEdJ6LvDpNn\n",
       "GxHtIaIDRLRjLGUnSve5osLu5qLPrrn30EujZE2a8T7MRT00Go1mtpCzMQ1v3e8fQq2O1wJgJxG9\n",
       "wMyHU/IUAvg7ALXMfJ6ISrMtOxkceWXd/ZGarncKF3WNGCIdQCWAN5vq63om8/kajUYz28hlT+MG\n",
       "ACeY+Qwz2wCegppDkcrXAfycmc8DADNfHkPZCdF+pLy8/2L+9WvvP/DqKFn9UC68upeh0WjmPbkU\n",
       "jWoMDbVx3ktLZSWAYiL6FRHtIqJvjKHshDj+xpoHS1Zcej2vbGBwlKwVAJqa6uvGPdiu0Wg0c4Vc\n",
       "utxmM0BtAdgC4A4AIQDvE9EHWZYFAHz8z489kDwuWbrp6JJbv3xstDItuxcuinUFV3zqmx/8dJSs\n",
       "eVCr+32SbX00Go1mpkFE2wBsm4x75VI0WqCWVE1Sg2tnUjcDuMzMUQBRInoLwCYv32hlAQBbv/G9\n",
       "F8dasVM7Vn6pfN2FRn9+IjFK1lIA/6xX59NoNLMZZt4BYEfynIgeG++9cikauwCsJKIlUKE9HgLw\n",
       "cFqe5wH80Bv49gO4EcBfAziWRdkxE+/z+Q69eN3n7ahVtvb+A++Okr0IStRGCiui0UwrRDQhl3PN\n",
       "3IeZaTLvlzPRYGaHiL4DoAmAAeDHzHyYiB71rj/OzEeI6FUA+wBIAD9i5kMAkKnseOsS6/X7jry8\n",
       "/nOXj5fdFYzETq3/4r4fmn53tPkWEQBPNdXX6R+lZkYz2Y2CZu6Qi5cKYp69bSIR8d1//tKjw12P\n",
       "9QT8RxrXb7t8ouyuYNHgsaW3nXy56v9v78yjpajuPP75ggqCqAEUHWOAiWCiA4yREKOO4hKjPqNx\n",
       "iVGMgifGZQa3M46OMe+oMXFiEh0TjUvivmQU9w3EDQ0SNwTcd0GPK4sbPJew/OaPe9tXNN3vVb/u\n",
       "qtev3+9zTp1TXXWrvreqbtev7vb7jXo7jbPB9YG5U5ubrq1dbh2n9kgyNxpOOcqVj2rKTSP4nlqF\n",
       "zz5cs/eLkzffYdFrA3fq07/lxX/Z+6lzNhjx7jspDxfQG7gvwyw6juN0SRrKaLQs6rPmS1M22/GD\n",
       "1wfu2GdAy3Mj9p3z+0Gbv/dehafZkBBkqT23Io7jON2OhjAaSxb07fPSlM12+nDegLF911vyzMj9\n",
       "Z521/jfmz+/AqVYjzF15qMZZdBzHaQi6fJ/GgK/Pn/zhG/23W2v9xXM22enluwcOW7CgilNuTHAX\n",
       "Mq1WeXScLPE+jfpB0qbA9YRQ0z83s/M7OUuZ9Gl0eS+3Sz9fvd8W42aeudWRM66u0mAU3IU8WqOs\n",
       "OY4TkfSgpA8krZGjpiS9Lum5EvvmSdqpaNsESdMTv9eQdJqklyUtkTRX0qWSBpeRPBG438zWroXB\n",
       "iNpXl9k3MXrR+FzS5dVqVUKXNxpbHTHjmgGbLFxUg1O5uxDHyYA432oMMB/YM0fp7Qgfg+tJGl20\n",
       "z2jf88SNwB6EOWJrEyYezyR4sCjFYDoYpC3OVSumrfy9DZwBXNYRvWro8kajRri7EMfJjkMIoxGv\n",
       "BsYDSOol6SNJmxcSSVpP0qcJb9cnSnpH0luSDpO0QtI/V6A7HriJMIl4fCUZlrQzwcv2Xmb2pJmt\n",
       "MLNPzOxCM1vlRS3pAYKbjvMlfSJpE0nrSLpK0vxYszlFkmL6CZJmSDpH0kKg1Aztss1HZnaLmd0G\n",
       "1OKDuSLcaAQGAndNbW5a2tkZcZwG5BBCW/8k4PuS1jOzLwgv9KSnh/2BB81soaRdgeMJX/XDCC/k\n",
       "1B2wkvoA+yZ0D5C0egV53hl4zMxSReo0sx2B6cB/xOapV4HzgH7AUGB7wn04NHHYGOA1wrywMyvI\n",
       "W5Lc+7PcaLS6C3mpszPiOFkgYbVYOqatbQkeqm83s1cIzTcHxd1/BQ5IJB8Xt0EwIJeZ2QvRN92p\n",
       "VPaC3Af4xMxmAA/EbU0VHD8AqHS4PsQ8xuamHwMnm1mLmb0BnA0cnEj7jpn9KdZiPu+AFlQZubQj\n",
       "uNEI7kKmTG1uWtHZGXGcLDBDtVg6KD8euMfMFsffN9DaVPQg0EfSmNjvMQq4Je7bkFXDI1SqezOA\n",
       "mS0HbmXlJqplBC/bSVYHCq0Ni2IeKqXwEh8Yz/dGYt+brBziIXl9HSX3mkZDzNOogvWB56Y2N83r\n",
       "7Iw4TqMhaU1CjaGHpHfj5l7AupJGmtnTkiYRmqjmA3eYWUtM9y6rerpOq/tVYEfg25L2j5v7AL0l\n",
       "9TezDwgv8KFFhw4F5sX1+4BjJG2UtomqiIUEAzQEKPjN+xorG7/2aglpahFe08gRdxfiONnyQ8IX\n",
       "/TcJtYhRcX06oX0fWpuokk1TEPohDpX0jdg/0VyB7sEE79TDE7rDCS/scTHN9cBxkjaNQ3NHE/ob\n",
       "rgMws/uAe4FbJH1L0mqS+kk6UtKhlEfx+OXxGn4taa04TPd44JoKrkMEg9tLUu+49ILQ/CWpN+HD\n",
       "v2dMU2oEVs3pzkZjA9xdiONkySGEfom3zGx+XN4HzgfGSephZo8DSwhNQVMKB5rZ3cAfgWmEUAmP\n",
       "xF1fAEj6uaTJbehekNAs6F5Eq7H6C3A5cAdh5OSVhAl59yTOsx8wmWBgPgKeIQSNu7eNa05++R8N\n",
       "tACvEwzltVGzkC5NTeNA4DPg07i8Evc1x98nAT+JaU5p53w1ocvPCG/Ly20brEYwGmdPbW76uMbZ\n",
       "cpzc6C4zwiV9k/DSXsPMvP8xJT4jvHZsCExzg+E49YukvWOzy1eAswgjsNxgdDLd0Wisi7sLcZyu\n",
       "wOHA+8CrhE7lozo3Ow5kPHoqTtA5lxB97xIzO6to/1jCbM3X46abzeyMuG8e8AmwHFhqZmM6mI2e\n",
       "QH9gTUIb4ZuEWoa7C3GcOsbMduvsPDirkpnRiD355xNmVr4NPCHp9hJhWx8ys1L+aAwYG4fHVUpv\n",
       "gqHoSfhCeQF4DnhjanNTS1sHOo7jOOXJsqYxBnjVzOYBSLoO2IvWMcsF2uqMqaSjZh2CUzGAj4GH\n",
       "CSMN3p7a3LSsgvM4juM4ZcjSaGzEqjM6v1OUxoCtJT1FqI2cYGbPJ/bdJ2k5cLGZ/aWMztdobXZ6\n",
       "kDA5Z+HU5qauOyzMcRynTsnSaKR5ac8CNjazTyXtRpjqPzzu28bM3pW0HnCvpBfNbHrxCab/72Hr\n",
       "frHkw49XLP1iI+AVM6smpobjOE7DEfuPx9bkXFnN05C0FXCame0af58MrCjuDC86Zi6wZXE/hqRT\n",
       "gSVmdnbR9m4xRt1xyuH/Aactuto8jZnAMElDYrSuHwO3JxNIGpTwLz+GYMQ+kNRHUr+4vS+wC2Fi\n",
       "j+M4Tl0SXZLMifE0JnZ2frIiM6NhZsuAicBUgjvk683sBUlHSCrM4t4PeEbSHMLQ3IKb5A2A6XH7\n",
       "Y8CdRdP7HcfpQsjDvXYk/yXDvcZ8XRqv4RNJs+P0hlzIdJ6GmU0h4U8mbrs4sf4n4E8ljnsd+Ncs\n",
       "8+Y4Tj6oNdzrm4RwrzfmJF0I97qWpNFmNjOxL224138i+H+aTYjweRAhMFSpMKuDgb93JKOSekYn\n",
       "h0nK5W81wr3czszelNQETJI0IsbtyJTuOCPccZx88XCvNQz3amafmtnpZvZm/H0XMJfgTDFz3Gg4\n",
       "jpM1Hu41u3CvSBpEGHW6SjNcFnT3IEyO0/DodNVkiKSdWvloG60c7nWxpEK413MJ8TMuBn4Rk48D\n",
       "LozrX4Z7jec5ldZYGGn4MtxrIs5EE2FYfxpqFe51VAws1SKpEO61UFN5JzbRA3Qo3Gs0hNcCV5jZ\n",
       "yx05R6W40XCcBqcjL/saUi7c67kkwr0SIvcVh3t9PHGeqsK9SiqEey0YjTThXodVqAk5hnuV1IPQ\n",
       "5Pc5YdBRLrjRcBwnE+ThXjML9xr7Ri4F1gN2L9GJnhnep+E4TlZ4uNeMwr0SmvG+AewZ+4dyw42G\n",
       "4zhZ4eFeMwj3KulrhFgjo4D3JC2Oy4HlT1U7uny4V3eh4HRnust/QB7utUOUKx/VlBuvaTiOU5fI\n",
       "w73WJW40HMepVzzcax3izVOO04Xx/4DTFt485TiO43QqbjQcx3Gc1LjRcBzHcVLjM8Idp4sj1ca3\n",
       "lOOkIVOjET1Vngv0BC4pDvWqELf2NsLkF4CbzOxXaY51HAe8E9zJm8yap6KXx/OBXYHNgAPjBJ1i\n",
       "HjKzLeLyqwqPzZVo5FzP9bq9XiNfW3fQq4Ys+zTGAK+a2TwzW0rw6bJXiXSlvpTSHps3Y13P9Vwv\n",
       "dy3XqyOyNBobsbLr37dY2S0wBN8qW0t6StJkSZtVcGxnMMT1XM/1ctdyvToiyz6NNJ1zs4CNzexT\n",
       "SbsRfN0PzzBP1TLE9VzP9XLXcr06Ikuj8Tar+sNfKZBKIjALZjZF0gWS+sd0bR5bIO+RI67neq6X\n",
       "v5br1Q9ZGo2ZwDBJQ4B3CKEPV3LdG2Pbzjczi9G7ZGYfSGr3WPCRI47jOHmTmdEws2WSJgJTCcNm\n",
       "LzWzFyQdEfdfTPBXf5SkZQRf8Qe0dWxWeXUcx3HS0aUdFjqO4zj54m5EHMdxnNS40XAcx3FS06WN\n",
       "hqShki6RdENiW19JV0r6s6RxbR1fhe5XJd0s6VJJJ2WhUaQnSb+W9EdJh7R/RNV6fSU9Iakpa62o\n",
       "t1d8XtdJ+l7GWpmXjyK93K4t6uX27DqhXObyvyt+r2RdZkroZVpm2nhvpio3XdpomNlcMzusaPM+\n",
       "wCQzOxzYMyPpEQQ/WT8FtshII8kPCZMb/0GZocc15kTg+hx0ADCz2+LzOpIwUi5L8igfX5LztUG+\n",
       "zy7vcpnL/67EeyXTMlOsl3WZKfPeTF1u6tpoSNpY0jRJz0l6VtIxKQ5LziZfnpHu34HDJd0P3N0R\n",
       "jQr1hgMzzOwEqgx52Z5W/LJ5HlhQjU4lmgl+QfA5lqVm1eWjQr0CmV9bFs+uLT1qWC5T6uX9vytQ\n",
       "kzLTgfdZVWUmrV6l5abeXaMvBY43szmS1gKelHQvMBr4FvA7M3un6JjCxMCn6bhRbE93AfALM5se\n",
       "q3hXdFAnrd5s4LOYdkXGWusAHxMcRX4mabJVP8Su3ecIHAtMMbM5VWq1p1mL8pFW7x7gJeA35HNt\n",
       "2wN9qe2za0vvLUItA6ovl2n0vk8O/7sSw/trVWZS6UkStSkzaa+vsnJjZl1mIbgZ2Snxuz9wEfAK\n",
       "cFLc1ge4DLgAODAj3ZHAjcCFwG9zuM41gUuAPwJHZamV2D4e2D2n53g0YTLohcARWWpmUT7a0Ns5\n",
       "z2vL49kV3cvMymUZvVz+d4n3yqvASVmVmRJ6rwD/nVWZKaN3UqXlpsvM01CYHf4QsLmZLWlU3Tz1\n",
       "OuOedgfNRn6GjarXqNeVhV5d92kUiFWrG4FjczYYuermqdcZ97Q7aDbyM2xUvUa9rqz06t5oSFod\n",
       "uAm4xsxubVTdPPU64552B81GfoaNqteo15WlXl03T8UOoSuBRWZ2fKPq5qnXGfe0O2g28jNsVL1G\n",
       "va7M9WrdmVTLBdiWMCpjDmEE0Wxg10bTzVOvM+5pd9Bs5GfYqHqNel1Z69V1TcNxHMepL+q+T8Nx\n",
       "HMepH9xoOI7jOKlxo+E4juOkxo2G4ziOkxo3Go7jOE5q3Gg4juM4qXGj4TiO46TGjYZTNZI2UIgy\n",
       "9qqkmZLukjSsRLpekh6KM1WzzM8ESedVeMyWkv7QAa3Bkg6s9jxdCYXIct/swHF7SmrOIk9OfrjR\n",
       "cKoiGoBbgAfMbBMzGw2cDAwqkfwg4E4rmlEqqdZxXSqasSppNTN70syO7YDWUODLEKBVnKemSOqZ\n",
       "4en3JsReSE3Mzx3AvtEfktNFcaPhVMsOwD/M7M+FDWb2tJk9XCLtgcBtAJLGSpou6TbgWUk9JP1O\n",
       "0uOSnpJ0eEzXQ9IFkl6QdE+sxewb982T1D+uj5Y0Lep8WZOR9ANJj0qaJeleSevH7adJulrSw8BV\n",
       "kraXdEfcN1nS7Lh8JOngWKP4m6Qn4/LdKPEb4N9i2uPidRXO01/SrfF6HpE0IqF9mUJUtdckHV3q\n",
       "xkpaIukchahr90kaGLf/LN6nOZJulLRm3H6FpIskPQqcJenbkv4er32GpOEx3YSYr3skzZU0UdIJ\n",
       "Md0jkr4S031d0pRYe/ybpE0lbQ38APhdvOahpdKVyk/8WHgE2CVNwXLqlKz8nvjSPRbgGOCcFOl6\n",
       "Au8mfo8FlgCD4+/DgVPiei/gCWAIsB9wV9w+CPgA2Cf+ngv0j+ujgWlxfQJwXlxfN6F5GPD7uH5a\n",
       "1OiVyM8dRXnekuC3px8h4FAh7TDgibi+ffK45HmA84DmuL4DMDuh/TCwOjAAWAj0LHHPVhCD/gDN\n",
       "iWvqn0hzBjAxrl8B3E6rI9J+hfMSAkLdmLg/rxCitQ0kRGo8PO47h+BCG+B+YJO4/h3g/rh+eeEZ\n",
       "tJNupfzEbYcSDEinl11fOrbUe7hXp/5J2xQ0EFhctO1xM3sjru8CjJC0X/y9NuHlvA0wCcDM3k/U\n",
       "JtKysaRJwAbAGsDriXzfbmZflDooftVfBfzIzBZLWgc4X9IoQpzoQp9NW/0z2wD7xLxPkzRAUr+o\n",
       "fZeZLQUWSZpPMIjFoYtXANfH9WuAm+P6CEm/IoTmXYvWeNkG3GDx7QysS6hFbRL3Jf/v08ysBWiR\n",
       "9BGh6QjgGWCkpL7A1sANau2CWiN5i+J9Wgv4bpl0xfkhXuOupW6W0zVwo+FUy3OE2kAail+wLUW/\n",
       "J5rZvSsdIO1edFxyfRmtTay9y2ieR6hd3Clpe8JXfoFPS2YytL//H3C6mT0fNx9PqCkdHPd/XkZv\n",
       "ldOV2f6PxPpy2v8vilYDfQWwp5k9I2k8oXZTIHlNZxC++veWNBh4MLEvaSxXJH6viHnpAXxoZluU\n",
       "yU8hLz2Aj9pIV3yPe1Bhn5NTX3ifhlMVZvYA0EvSzwrbJI2UtG1R0oWEr+JyTAX+vdApLmm4pD7A\n",
       "DELnqSQNIjQHFZhHaJYC2LfMedem9Qt+QmJ7WzWE3wBPm9mkovO8F9cPITS3Qag99StznumEzn8k\n",
       "jQUWmNnidrST9AB+FNfHxfNBuI/vKXQo/4TyL+HktR+aUlMAMZ9zCzW/eP9HxjSL47kxs0/aSFeK\n",
       "DYE32tjv1DluNJxasDews8KQ22eBXwPvJhOY2XJCh/emhU2s/LK7BHgemCXpGeBCwov5JuCtuO9q\n",
       "YBahDR7gdOAPkp4g1DoK50ue+zRC08lMYEGZNMW//xP4XqIzfA/gAmC8pDnApoT+GICngOWxU/q4\n",
       "EtpbSnoKOBMYX0a7HC3AmHg/xgK/jNubgccI/SIvFB2TPO9vgf+RNItwL9u69lL34SDgp/GanwX2\n",
       "jNuvA/4rDggY2ka64nMDjAH+1sY1O3WOx9NwckPSBGCQmZ1V4XF9zaxF0gDCy3JrM5ufRR7rCUmL\n",
       "zaxcLabLIakHweiPNrNlnZ0fp2N4TcPJk78CTVLFk/vulDSb8IX6y+5gMCKN9kW3B2EElxuMLozX\n",
       "NBzHcZzUeE3DcRzHSY0bDcdxHCc1bjQcx3Gc1LjRcBzHcVLjRsNxHMdJjRsNx3EcJzX/DyXfpdp6\n",
       "YMhbAAAAAElFTkSuQmCC\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1130f4a10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "l1_mean = numpy.array([x.mean_validation_score for x in tree_grid_searcher.grid_scores_ if x.parameters['penalty'] == 'l1'])\n",
    "l2_mean = numpy.array([x.mean_validation_score for x in tree_grid_searcher.grid_scores_ if x.parameters['penalty'] == 'l2'])\n",
    "l1_std = numpy.array([x.cv_validation_scores.std() for x in tree_grid_searcher.grid_scores_ if x.parameters['penalty'] == 'l1'])\n",
    "l2_std = numpy.array([x.cv_validation_scores.std() for x in tree_grid_searcher.grid_scores_ if x.parameters['penalty'] == 'l2'])\n",
    "print \"another method max auc =\", l2_mean.max(), \"with c = \", c[numpy.where(l2_mean==l2_mean.max())[0][0]], \"2^\", math.log(c[numpy.where(l2_mean==l2_mean.max())[0][0]], 2)\n",
    "plot.fill_between(c, l1_mean - l1_std, l1_mean + l1_std, color = 'steelblue', alpha = .4)\n",
    "plot.plot(c, l1_mean, label='Avg. AUC for L1')\n",
    "plot.xscale('log', basex=2); plot.legend(loc=4);\n",
    "plot.xlabel('C (regularization parameter)'); plot.ylabel('AUC'); \n",
    "plot.fill_between(c, l2_mean - l2_std, l2_mean + l2_std, color = 'springgreen', alpha = .4)\n",
    "plot.plot(c, l2_mean, label='Avg. AUC for L2')\n",
    "plot.xscale('log', basex=2); plot.legend(loc=4);\n",
    "plot.xlabel('C (regularization parameter)'); plot.ylabel('AUC'); "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "TOP 10 pos\n",
      "fun, importance = 0.58220\n",
      "great, importance = 0.43094\n",
      "american, importance = 0.36546\n",
      "well, importance = 0.36121\n",
      "memorable, importance = 0.35830\n",
      "jackie, importance = 0.31934\n",
      "excellent, importance = 0.31870\n",
      "solid, importance = 0.31402\n",
      "job, importance = 0.31284\n",
      "mulan, importance = 0.30724\n",
      "\n",
      "TOP 10 neg\n",
      "bad, importance = -0.69696\n",
      "unfortunately, importance = -0.56733\n",
      "worst, importance = -0.55011\n",
      "nothing, importance = -0.44924\n",
      "plot, importance = -0.41437\n",
      "only, importance = -0.40243\n",
      "poor, importance = -0.39973\n",
      "script, importance = -0.39759\n",
      "boring, importance = -0.35820\n",
      "awful, importance = -0.35088\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "\n",
    "print '\\n', 'TOP 10 pos'\n",
    "for idx in numpy.argsort(model.coef_)[0][::-1][:10]: # последние 10 с максимальным коэфициентом\n",
    "    print '{}, importance = {:.5f}'.format(words[idx], model.coef_[0][idx])\n",
    "print '\\n', 'TOP 10 neg'\n",
    "for idx in numpy.argsort(model.coef_)[0][:10]: # последние 10 с минимальным коэфициентом\n",
    "    print '{}, importance = {:.5f}'.format(words[idx], model.coef_[0][idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vect_WOSW = CountVectorizer(stop_words='english')\n",
    "matrix_count_WOSW = count_vect_WOSW.fit_transform(all_texts).toarray()\n",
    "\n",
    "X_train_WOSW, X_test_WOSW, Y_train_WOSW, Y_test_WOSW = cross_valid(matrix_count_WOSW, target)\n",
    "\n",
    "model_WOSW = LogisticRegression()\n",
    "model_WOSW.fit(X_train_WOSW, Y_train_WOSW)\n",
    "predict_WOSW = model_WOSW.predict_proba(X_test_WOSW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 39355)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix_count_WOSW.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression ( l2 ) AUC =  0.931306706992\n",
      "Logistic Regression ( l2 WOSW ) AUC =  0.917152197118\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAecAAAEZCAYAAACtle09AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYZFV57/HvjwF1TCA2QY0BFC94PWogAZJ4YRAvgIl4\n",
       "hYNiVPToScSYGI/wGBOGmIgcjUc93hBFjCJEI0lGQzCKjCDXoNw0YJgoyoCKMKOiooK8+WPvZoqe\n",
       "vlTPdHXtqvp+nqee6b33qrXf2jT19lp77bVSVUiSpO7YZtgBSJKkuzI5S5LUMSZnSZI6xuQsSVLH\n",
       "mJwlSeoYk7MkSR1jctZISXJtkv2HHYckDZLJWaOm2tfISvLiJL9IckuSHyS5IsmzZpS5e5Ljknwz\n",
       "yU+S/GeS185S19OSnJPkh0luTLI2ye8v36eRNAgmZ2mGtAZ8mvOqanvgXsC7gI8l2bHn+CeA/YAD\n",
       "gV8GXgi8PMk7euJ8LvBx4GRg56q6D/CXgMlZGnEmZ42stnX59iTXt6//l+RuPcdfl+SGJOuTvCzJ\n",
       "HUkeNEdda5P8dZLzgB8DD0zy8CSfTXJzkquTPK+n/K8m+VTb8r24fe+5iwkfoJop+j4K3B14UFv3\n",
       "/sBTgOdU1X9U1R1VdRFwOPDKJA9q/3h4G/BXVXVSVd3S1ndOVb18EXFI6iCTs0bZnwN7A49tX3sD\n",
       "bwBIcgDwp8D+wO7AKhbuDj8ceBlNS/Vm4LM0ifPewP8E3pPkEW3ZdwO3APcFXgT8QR/1bybJCuAl\n",
       "wPeBr7W7nwJcWFXX95atqouB9e1nehiwC/APiz2npO4zOWuUPZ+m5XhTVd0EHEvT/QtwCHBSVV1V\n",
       "VbcCx9C2VudQwMlt+TuAA4BvVNWH25brZcDpwPPahPps4Jiq+mlVXQV8eIH6Z/rtJBuBW4G3AL8/\n",
       "3foFdgK+M8f7vt0e/9WebUljxuSsUfbrwDd7tr/V7gO4H3Bdz7H1fdTXW/4BwD5JNk6/aP4YuC9N\n",
       "ctx2C+rvdWFVTQFTwBrgqJ5j32vjn839gJvaF/OUkzTCTM4aZTcAu/Vs3x+Y7gr+NrBrz7Hen+fS\n",
       "2y39LeALVTXV89q+ql5Jkxhv34L6Nz9h1Y+BPwT2TbJvu/tzNH8Y7NJbNsk+7Xk+D/wnzR8Hz92S\n",
       "80rqNpOzRtmpwBuS7JRkJ5qRyh9tj30ceEk7qOuewF/0UV9vt/SngYcmOTzJdu1rryQPr6pf0HRx\n",
       "r06yMsnDabrT70zu7QCzY/r5EFW1EXg/cHS7fRZwFvDJJI9MsiLJbwMfAd5TVf/VDiR7DfAX7aNZ\n",
       "OyTZJsnjk5zQz3kldZfJWaPsr4FLgCva1yXtPqrqTOCdwNk0rcwL2vf8bJ767kyuVfUj4Kk0A8Gu\n",
       "p2mJHwdMjwY/EvgVmnvDH6b5Q+HnPXXtAnxxnvPMHDz2dmC/JI9pt5/Txn4mzcCzjwAfqKpX9cT4\n",
       "SeBQ4Ig2xu8AfwX80zyfUdIISPMH+IAqT04Cng7cWFWPnqPMO2me5fwJ8OKqunRgAWlitaOsrwTu\n",
       "1g74Wur6jwfuU1UvabujT6uqxy/1eSRNhkG3nD9EM+p1VkkOAh5SVbsDLwfeO+B4NEGSPKt9FnoK\n",
       "OB5Ys1SJOcnDkjymna9kb5rW6z8CVNV6E7OkrTHQ5FxV5wIb5ynyDJouQdpJFu6V5L6DjEkT5eXA\n",
       "d4F1wG00A6+WyvbAJ4EfAacBb62qNUtYv6QJtu2Qz78zmz+OsgvNF6q0VarqwAHWfQnN5CaStOS6\n",
       "MCBs5sQNI72ogSRJW2vYLefruevzobuw6TnVOyUxYUvSFqiqQS/iogEYdnJeQ/NIymntc5zfr6pZ\n",
       "u7T9BWskWV1Vq4cdRxd4LSDHZgMwxdk0a1h1x8Y6pnZcuNgmCc1n2Wqr29eS2ljFoj5PF9iwGV0D\n",
       "Tc5JTgX2BXZKch3N/MbbAVTVCVV1RpKDkqyjWQnoJYOMR+q6O5Ntv26dguM3AKvhC6sHFNUWmcrq\n",
       "Rd+i2li1qPnJZ5Ucu7pq9eqtrUcapoEm56o6rI8yRw4yBmkU5OipO1j5/WxKtrOavfX2ZhOSNG6G\n",
       "3a2txVs77AA6ZO2wA9hSm7eQp6hj2ls3b96iKtdufVRjY+2wA5C21kBnCFsqScp7zho1CRs4ascp\n",
       "Vs7yqP/mLeSRvKepbvO7c3SZnKUBSShWZ1OLWFpmc313OlCsO+bKbXZrS7NY1Mjho3Zk1tbxamD+\n",
       "GfKkobHBM3zz/ZFkclanLd3jNYt258jhPkZQL/qxIUmaj8lZnTJLMl6Sx2u2OJ4mMds1LWlZmZw1\n",
       "cIts/S5LMl7E88S2iiUtO5OzBqYnKQ+19TuHKVvDUrckuRZ4aVWdNexYhq0LC19oDCRsSKjeF0AV\n",
       "8REhSX0qoJK8KMmXkvwgyXVJjk+yYtjBLSdbzloqU11rHS/Qde0oaqmbAqwE/hi4CLgPzToMrwWO\n",
       "H2Jcy8rkrHkt4n7x0JLdPEl4o13X0sipqnpfz/YNSU6ha0u7DJjJWQvpXIt4Wk9SNglL421f4CvD\n",
       "DmI5mZw1p7bV3LnuX5OyNFjTY0a21tKsMpYjgD2BI7Y+otFhctas2sTMMAZz9Tnph0lZGpCu9JYl\n",
       "eSbwJmD/qppzubZxZHKeEFsw09ZWLcSw6HWJZ57b5CtNtCQHAO8HDqqqrw47nuVmcp4APa3g5Zzc\n",
       "wwQraYskeRJwCnBwVV0y7HiGweQ8GQY2qGuWFrJJWdLWCPDnwPbAvyZ3fp2cU1VPH1pUy8wlI8fc\n",
       "lt47dnpLafTNt2Sk36nDN99/B1vOY2jG/eVF3Tu2W1qShs/kPMLmGeS14FzWTtwhSd1lch4RcyTi\n",
       "rVlQwoUfJKmjTM4dNIBEPOvArS2tS5I0WCbnblr06Gon7pCk8WFy7pDe9Y/nLTd7Ijb5StKYMDkP\n",
       "2Swjq/tJsN4vlqQxZnIeosXM3DWjtez9YkkaYybn4err3nKbmLG1LGncJbkWeGlVnTXsWIZpm2EH\n",
       "MKn6XY6xJzE7C5ekSVDtiyQvSnJJkh8kuS7J8UlWzPamJCckeU/P9nZJfjzHvr3b7bsnOS7JN5P8\n",
       "JMl/JnntjHofleTfktycZGMbz4Htsa8lOaSn7OOS3DHLvh8mWVS+teU8PHdpNS8wKYiJWdIkWgm8\n",
       "GrgIuA+wBngtcPwsZb8AvL5n+7eAbwJPmLGvgC+1259o6z0QuBrYC/hIkl2r6tVtmU8B7wYOopn3\n",
       "e6/23+lzPhH4eLv9xLaemfvOr6o7FvG5Tc7DkGOzgdWQY++yoLmjrSWpR1W9r2fzhiSnAPvNUfxc\n",
       "4BFJdmzXfn48cBrw4iS/WlU30yTq86vqF0n2B54CPKSqrm/ruCjJ4cD5Sd4B/BDYDTixqm5vy5zf\n",
       "c85zgNf1bD+e5g+HP+vZ94S23KKYnJfBnSOyj9oRVm6EW6fg+A1btV6yJE2gfYGvzHagqq5LMt1S\n",
       "/meaFuvbgIe2+/6p3TedKJ8CXNiTmKfruTjJemD/qjoxyTrglCQfaMt/t6f4ucCHk9yLJpH/FvAc\n",
       "4I09+34XeNNiP6jJeYBy9NQdrPx+WH3nrk1d1G8eUlCStIAcmyVZrnApewOTHAHsCRwxT7EvAPsm\n",
       "WQPsDVxAk5yfmOSfaRLldJf4TsB35qjn2+1xaFrqRwN/CzwwyRdpBqytq6pvJvkWTdK/Drimqn6a\n",
       "5LyefXej6ZZfFJPzIK38fuyqljRquva9leSZNK3P/dsu67mcA7wSeDTw9Z5E+b/afSvZlCi/Bzxk\n",
       "jnruB9wE0LasX9XGsQvwfuDvaBL99DmfCHyLTa3yL/bsu6iqblvM5wWT81bZbA7s6W7rabfeq/uL\n",
       "ZUtShyU5gCYhHlRVX12g+Llt2ae3PwN8Fdi13XdxVf283f854E+S7FJV63vOt09b/vMzK6+q9e3o\n",
       "74/17D4HeAXN4LOTeuJ4Ubtv0febweS8te4cce0jT5K0tJI8CTgFOLiqLlmofFWtS3IjzQjvl7X7\n",
       "KslF7b4Te8qeleQs4JNJXgJ8jWYk9t8B76mq/2rvG/9pu+8bwI403eoX9Jz2HOAE4AHAS9t9VwIP\n",
       "al+9g9r65nPOfUjYkFAzX8DGHJsN0/dnTMyStKTeAGwP/GuSW9rXvyzwni/Q3C8+r2ffucC92bwV\n",
       "+xzgbOBM4BbgI8AHqupV7fGf0yTdzwE/oEm6twIvnq6gqq4BbgS+XVU/bPcVTff59tx1dHff0tQx\n",
       "GG13xNuBFTQf+PgZx3cCPgr8Gk0r/q1VdfIs9VTV8t8D6V2IYq6R1Tk21bX7M5IEc393Dus7VXc1\n",
       "33+HgbWc21lc3gUcADwSOCzJI2YUOxK4tKp+A1gF/G2SoXa197aSoZn32keeJEnLaZDd2nsD66rq\n",
       "2nak2mnAwTPKfBvYof15B+Dmnge9l13vQhQmZUnSsAyylbozzTNe09YD+8wocyLw+SQ30PTNH8KQ\n",
       "9CRmE7IkaagGmZz7uZn9euCyqlqV5MHAZ5M8tqpumVkwyeqezbVVtXZpwjQxSxoPSVbR3CLUiBtk\n",
       "cr6e5lmxabvStJ57/S7wNwDtsPVvAA8DNhsyX1WrBxHkYhPzLAtUuLaypE5oGy1rp7eTHDO0YLRV\n",
       "BpmcLwF2T7IbcANwKHDYjDJXA08GzktyX5rE/PUBxnQXW9hinnJ0tiRpkAaWnKvq9iRHAp+heZTq\n",
       "g1V1VZJXtMdPoJmO7UNJLqcZnPa6BaZmW2p3WbZRkiZFsjTzZ2swBvqc81IZ1DN5CbXY5OxzzZJG\n",
       "hc8zjy5nCJMkqWMmLjnPmGTEwVySpM6ZqIUveicZGXYskiTNZaKSM1s5AKx9jMrWtiRpoCauW3tL\n",
       "uSSkJGm5TEzLue3S3ppWr883S5KWxcQkZ7agS3vGbGB2Z0uSlsUkJedF6enGtrUsSVpWJucZelrL\n",
       "G72/LEkaholIzou83+y9ZUnSUE1EcsY5tCVJI8RHqXr4HLMkqQsmpeXcL7u0JUlDZ8tZkqSOMTm3\n",
       "7NKWJHWF3dqb2KUtSeoEW86SJHXMWLec2+ebp5inu7p30pHlikuSpPmMdXKmv+eb7c6WJHXKRHdr\n",
       "OwhMktRF495yXoitZklS50x0y1mSpC6auJazazRLkrpu4pIzdmVLkjrObm1JkjrG5CxJUseYnCVJ\n",
       "6hiTsyRJHTO2ybmdutPR2JKkkTPOo7WnqsiMR6fAhC1J6rhU1bBjWFCSqlrc4085esdi5UaAjXVM\n",
       "7TiYyCSpu7bku1PdML4t55Ub8XlmSdIo6uuec5J7JnnYoIORJEl9JOckzwAuBT7Tbu+RZM2gA9sS\n",
       "CRsSKqH7ffWSJM2hn5bzamAf2oFUVXUp8KABxrQ1pqpIH2s4S5LUWf0k59uq6vsz9t3RT+VJDkhy\n",
       "dZJrkhw1R5lVSS5N8pUka/upd/Z6fHRKkjQe+hkQ9tUkLwC2TbI78MfA+Qu9KckK4F3Ak4HrgX9P\n",
       "sqaqruopcy/g3cDTqmp9kp225EO0pmwxS5LGQT8t51cBjwJ+BpwK/BD4kz7etzewrqqurarbgNOA\n",
       "g2eUeT7wyapaD1BVN/UbuCRJ46qflvPDqur1wOsXWffOwHU92+tp7l332h3YLsnZwPbAO6rqI4s8\n",
       "j13akqSx0k9yfluSXwM+Afx9VX2lz7r7GTG9HbAnsD9wT+CCJBdW1TV9nmOaXdqSpLGxYHKuqlVJ\n",
       "7gccApyQZAfg41X1xgXeej2wa8/2rjSt517XATdV1a3ArUnOAR4LbJack6zu2VxbVWsXil2SJkmS\n",
       "VcCqIYehJbCo6TuTPBo4Cji0qrZboOy2wNdoWsU3ABcDh80YEPZwmkFjTwPuDlzU1v0fM+qadwq6\n",
       "hJpuOffMpe20nZImmtN3jq4FW85JHknTan4ucDPw98BrFnpfVd2e5EiayUtWAB+sqquSvKI9fkJV\n",
       "XZ3kTOAKmsezTpyZmLfAlNN2SpJG2YIt5yQX0oy0/kRVXb8sUW0ew/wt502LXIAtZkkCbDmPspFf\n",
       "lSphA6tja1mSZjA5j645u7WTfKKqnpfkylkOV1U9ZoBxLcbUwkUkSRod891zfnX77+/BZo8pdb+5\n",
       "LUnSiJpzhrCquqH98Y/aWb7ufAF/tCzRSZI0gfqZvvOps+w7aKkDkSRJjfnuOf8hTQv5wTPuO28P\n",
       "nDfowPrRM22n950lSWNjvnvOHwP+FXgzzcQj0/edb6mqmwcdWJ+mqkiO9R64JGl8zJecq6quTfJK\n",
       "ZgwAS7JjVW0YbGiSJE2m+ZLzqcDTgS8x++jsBw4koj65EpUkaVzNmZyr6untv7stWzSL40pUkqSx\n",
       "tOBo7SSPS/LL7c8vTPK2JA8YfGiSJE2mfh6leh/wkySPpVnw4uvA3w00KkmSJlg/yfn2qroDeCbw\n",
       "7qp6F83jVJIkaQAWXDISuCXJ64HDgSckWQHMu5azJEnacv20nA8FfgYcUVXfAXYG3jLQqCRJmmB9\n",
       "LRmZ5NeAvWgeqbq4qm4cdGAzzr/ZsmcJ1UxAkg0AruEsSXflkpGjq5/R2ocAFwHPAw4BLk7yvEEH\n",
       "tghTJmZJ0jjp557zG4C9plvLSe4NnAV8YpCBSZI0qfq55xzgez3bN7P5+s6SJGmJ9NNyPhP4TJKP\n",
       "0STlQ2kWxJAkSQPQ74CwZwOPbzfPrap/HGhUm59/1gFhrM5GcDCYJM3GAWGja771nB9K88jUQ4Ar\n",
       "gP9TVeuXK7A+TdUx/uJJksbLfPecTwI+DTwH+DLwzmWJaAEJGxKKo6YKV6WSJI2hObu1k1xWVb/R\n",
       "s31pVe2xbJHdNZY7u2Z6nm8uW82SNDe7tUfXfAPC7pFkz/bnACvb7QBVVV8eeHSSJE2g+ZLzd4C/\n",
       "nWd7v4FEJEnShJszOVfVqmWMQ5IktfqZhESSJC2jkUrOCRtwhLYkacz1M0NYl0xVOXWoJGm89bMq\n",
       "1TZJXpjkL9vt+yfZe/ChSZI0mfrp1n4P8DvA89vtH7X7JEnSAPTTrb1PVe2R5FKAqtqQZLsBxyVJ\n",
       "0sTqp+X88yQrpjfa9ZzvGFxIkiRNtn6S8/8H/hG4T5I3AecBxw00KkmSJtiC3dpV9dEkXwL2b3cd\n",
       "XFVXDTasueXYbACm8JEqSdKY6me09v2BHwOfal8/bvctKMkBSa5Ock2So+Ypt1eS29t1oxcyVcdU\n",
       "XMNZkjSu+hkQdgYwvXTVPYAHAl8DHjXfm9r71O8CngxcD/x7kjUzW91tueOBM8FnmCVJ6qdb+3/0\n",
       "brcrU72yj7r3BtZV1bXt+04DDgZmdom/CvgHYK8+6pQkaewtevrOdqnIffooujNwXc/2+nbfnZLs\n",
       "TJOw3ztd/WLjkSRp3CzYck7yZz2b2wB70nRTL6SfRPt24OiqqiTBbm1Jkvq65/zLPT/fDnwa+GQf\n",
       "77se2LVne1ea1nOv3wROa/IyOwEHJrmtqtbMrCzJajgGzoaszqqqWttHDJI0MZKsAlYNOQwtgVTN\n",
       "3cBtB2v936r6szkLzf3ebWkGju0P3ABcDBw212NYST4EfKqqTp/lWNu4plgd6piyhS1JC5j+7hx2\n",
       "HFq8OVvOSbatqtuTPC7tf+HFVNy+90jgM8AK4INVdVWSV7THT9iqyCVJGlNztpyTfLmq9kzyPuDX\n",
       "gU8AP2kP12wt3EGx5SxJi2fLeXTNd895+j/oPYCbgSfNOL5syVmSpEkyX3K+d5LXAFcuVzCSJGn+\n",
       "5LwC2H65ApEkSY35kvN3qurYZYtEkiQBWzBD2FAdtSO4GpUkaczNl5yfvGxR9GvlRlyNSpI07uZM\n",
       "zlV183IGIkmSGqPVrS1J0gQwOUuS1DEmZ0mSOsbkLElSx5icJUnqGJOzJEkdMzLJOcdmA7dODTsM\n",
       "SZIGbmSSMzDF8RuGHYMkSQM3SslZkqSJYHKWJKljTM6SJHWMyVmSpI4xOUuS1DGjlpxdy1mSNPZG\n",
       "KjlX4VrOkqSxN1LJWZKkSWByliSpY0zOkiR1jMlZkqSOMTlLktQxJmdJkjrG5CxJUseYnCVJ6hiT\n",
       "syRJHWNyliSpY0zOkiR1jMlZkqSOMTlLktQxJmdJkjpm4Mk5yQFJrk5yTZKjZjn+giSXJ7kiyXlJ\n",
       "HjPomCRJ6rKBJuckK4B3AQcAjwQOS/KIGcW+Djyxqh4DvBF4/yBjkiSp6wbdct4bWFdV11bVbcBp\n",
       "wMG9Barqgqr6Qbt5EbDLgGOSJKnTBp2cdwau69le3+6by0uBMwYakSRJHbftgOuvfgsm2Q84Anjc\n",
       "rAXOhqzO6nZrbVWt3drgJGmcJFkFrBpyGFoCg07O1wO79mzvStN6vot2ENiJwAFVtXHWmvaDWlur\n",
       "BxCjJI2FttGydno7yTFDC0ZbZdDd2pcAuyfZLcndgEOBNb0FktwfOB04vKrWzVPX7ElbkqQxk6q+\n",
       "e5637ATJgcDbgRXAB6vquCSvAKiqE5J8AHgW8K32LbdV1d4z6qiqykADlaQx43fn6Bp4cl4K/oJJ\n",
       "0uL53Tm6nCFMkqSOMTlLktQxJmdJkjrG5CxJUseYnCVJ6hiTsyRJHWNyliSpY0zOkiR1jMlZkqSO\n",
       "MTlLktQxJmdJkjrG5CxJUseYnCVJ6hiTsyRJHWNyliSpY0zOkiR1jMlZkqSOMTlLktQxJmdJkjrG\n",
       "5CxJUseYnCVJ6hiTsyRJHWNyliSpY0zOkiR1jMlZkqSOMTlLktQxJmdJkjrG5CxJUseYnCVJ6hiT\n",
       "syRJHWNyliSpY0zOkiR1jMlZkqSOMTlLktQxJmdJkjpmoMk5yQFJrk5yTZKj5ijzzvb45Un2GGQ8\n",
       "kiSNgoEl5yQrgHcBBwCPBA5L8ogZZQ4CHlJVuwMvB947qHjGRZJVw46hK7wWm3gtNvFaaBwMsuW8\n",
       "N7Cuqq6tqtuA04CDZ5R5BvBhgKq6CLhXkvsOMKZxsGrYAXTIqmEH0CGrhh1Ah6wadgDS1hpkct4Z\n",
       "uK5ne327b6EyuwwwJkmSOm+Qybn6LJctfJ8kSWNp2wHWfT2wa8/2rjQt4/nK7NLu20wSk3YryTHD\n",
       "jqErvBabeC028Vpo1A0yOV8C7J5kN+AG4FDgsBll1gBHAqcl+W3g+1X13ZkVVdXM1rUkSWNrYMm5\n",
       "qm5PciTwGWAF8MGquirJK9rjJ1TVGUkOSrIO+DHwkkHFI0nSqEiVvcWSJHVJp2YIc9KSTRa6Fkle\n",
       "0F6DK5Kcl+Qxw4hzOfTze9GW2yvJ7UmevZzxLZc+//9YleTSJF9JsnaZQ1w2ffz/sVOSM5Nc1l6L\n",
       "Fw8hzGWR5KQk301y5TxlJuJ7c6xUVSdeNF3f64DdgO2Ay4BHzChzEHBG+/M+wIXDjnuI1+J3gF9p\n",
       "fz5gkq9FT7nPA58GnjPsuIf0O3Ev4KvALu32TsOOe4jXYjVw3PR1AG4Gth127AO6Hk8A9gCunOP4\n",
       "RHxvjturSy1nJy3ZZMFrUVUXVNUP2s2LGN/nw/v5vQB4FfAPwPeWM7hl1M91eD7wyapaD1BVNy1z\n",
       "jMuln2vxbWCH9ucdgJur6vZljHHZVNW5wMZ5ikzK9+ZY6VJydtKSTfq5Fr1eCpwx0IiGZ8FrkWRn\n",
       "mi/n6elfx3EgRT+/E7sDOyY5O8klSV64bNEtr36uxYnAo5LcAFwOvHqZYuuiSfneHCuDfJRqsZy0\n",
       "ZJO+P1OS/YAjgMcNLpyh6udavB04uqoqSdj8d2Qc9HMdtgP2BPYH7glckOTCqrpmoJEtv36uxeuB\n",
       "y6pqVZIHA59N8tiqumXAsXXVJHxvjpUuJeclnbRkxPVzLWgHgZ0IHFBV83VrjbJ+rsVv0jwrD839\n",
       "xQOT3FZVa5YnxGXRz3W4Dripqm4Fbk1yDvBYYNyScz/X4neBvwGoqv9K8g3gYTTzL0yaSfneHCtd\n",
       "6ta+c9KSJHejmbRk5pfrGuAPAOabtGQMLHgtktwfOB04vKrWDSHG5bLgtaiqB1XVA6vqgTT3nf9w\n",
       "zBIz9Pf/xz8Dj0+yIsk9aQb//Mcyx7kc+rkWVwNPBmjvrz4M+PqyRtkdk/K9OVY603IuJy25Uz/X\n",
       "AvhLYAp4b9tivK2q9h5WzIPS57UYe33+/3F1kjOBK4A7gBOrauySc5+/E28CPpTkcppGyOuqasPQ\n",
       "gh6gJKcC+wI7JbkOOIbmFsdEfW+OGychkSSpY7rUrS1JkjA5S5LUOSZnSZI6xuQsSVLHmJwlSeoY\n",
       "k7MkSR1jctZIS/KLdonE6df95yn7oyU438lJvt6e60vtpA6LrePEJA9vf379jGPnbW2Mkkafzzlr\n",
       "pCW5paq2X+qy89TxIeBTVXV6kqcAb62qx25FfVsdk6TxY8tZYyXJLyX5XNuqvSLJM2Ypc78k57St\n",
       "3yuTPL7d/9Qk57fv/XiSX5rrNO2/5wIPad/7mrauK5O8uieWf0lyWbv/ee3+tUl+M8mbgZVtHB9p\n",
       "j/2o/fe0JAf1xHxykmcn2SbJW5JcnOTyJC9fmisnqUs6M32ntIVWJrm0/fnrwCHAs6rqliQ7ARew\n",
       "+bzLzwfOrKo3JdkGuGdb9s+B/avq1iRHAa8B3jjPuX8fuCLJnsCLadYZ3ga4KMkXgAcD11fV0wGS\n",
       "TK8vXEBV1dFJXllVe/TUOd2VdVr7Wc5o549+EvAK4GU0cyPvneTuwBeT/FtVXdvn9ZI0AkzOGnW3\n",
       "9ia3JNsBxyV5As380r+e5D5VdWPPey4GTmrL/lNVXZ5kFfBI4Px2rvK7AefPcr4Ab0nyBuBGmrW0\n",
       "nwKc3q4GRZLTgScAZwJvbVvIn66qLy7ic50JvKNNzAcCX6iqnyV5KvDoJM9ty+1A03q/dhF1S+o4\n",
       "k7PGzQtolo3cs6p+0S4VeI/eAlV1bpu8fw84OcnbgI3AZ6vq+QvUX8Brq+r06R1Jnsxd18tNc5q6\n",
       "JskewNOBv05yVlXN1xLvjfGnSdYCT6NpQZ/ac/jIqvpsP/VIGk3ec9a42QG4sU3M+wEPmFmgHdH9\n",
       "var6APABYA/gQuBxSR7clvmlJLvPcY6ZC9efCzwzycr2PvUzgXOT3A/4aVWdAry1Pc9MtyWZ64/k\n",
       "vweOYFMrHJqVmP5o+j1JHtouDylpjNhy1qib+bjBKcCnklxBs+7vVbOU3Q94bZLbgFuAP6iqm5K8\n",
       "GDi1vZcLzT3oaxY6Z1VdmuRkmu5yaJZqvLztgn5LkjuA24D/PUtd76e5b/2lqnrhjLr/DfgITdf7\n",
       "7e2+DwD/ErKdAAAASUlEQVS7AV9O0/9+I/CsWeqVNMJ8lEqSpI6xW1uSpI4xOUuS1DEmZ0mSOsbk\n",
       "LElSx5icJUnqGJOzJEkdY3KWJKljTM6SJHXMfwMo8hHTs2YGOQAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13914d8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_curve('log reg, ROC', Y_test, predict, label='l2')\n",
    "roc_curve('log reg, ROC', Y_test, predict_WOSW, label='l2 WOSW')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "308"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logistic_reg(X_train, X_test, Y_train, **kwargs):\n",
    "    model_logistic = LogisticRegression(**kwargs)\n",
    "    model_logistic.fit(X_train, Y_train)\n",
    "    predicted = model_logistic.predict_proba(X_test)\n",
    "    return predicted, model_logistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_auc(Y_test, predicted):\n",
    "    fpr, tpr, _ = metrics.roc_curve(Y_test, predicted[:,1])\n",
    "    return metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEPCAYAAABlZDIgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYXFWd//H3x4RFlhAQBlkiQTYDAwiOkZEf0gpiUBQX\n",
       "NMCIggwyOlFGR0VwIcEFXFCYcRlANhdExQ2cyKJD4woBSQKBJBIIW1hlR1AI+f7+OKfITaW6u6r7\n",
       "3q6lP6/n6aer7nLuObeq7vfec+49RxGBmZnZSD2v3RkwM7Pe4IBiZmalcEAxM7NSOKCYmVkpHFDM\n",
       "zKwUDihmZlaKSgOKpGmSFkm6WdKxDeZvKOmnkuZLulrSToV5EyVdKGmhpJsk7VFlXs3MbGQqCyiS\n",
       "xgFfA6YBOwKHSJpSt9jxwHURsSvwLuC0wrzTgNkRMQXYBVhYVV7NzGzkqrxCmQosiYjbIuIZ4ALg\n",
       "wLplpgBXAETEYmCypE0kbQDsFRFn53nLI+LRCvNqZmYjVGVA2QK4s/D+rjytaD7wVgBJU4GtgC2B\n",
       "rYEHJJ0j6TpJZ0pap8K8mpnZCFUZUJrp0+VkYKKkucAMYC7wLDAe2B34RkTsDvwV+HhVGTUzs5Eb\n",
       "X2Hay4BJhfeTSFcpz4mIx4H31N5LWgrcCqwH3BUR1+RZF9IgoEhyR2RmZsMQESo7zSoDyrXAdpIm\n",
       "A3cD04FDigvktpKnIuJpSUcBV0bEE8ATku6UtH1E/BnYF7ix0Uaq2CmdQtLMiJjZ7nxUxeXrbr1c\n",
       "vl4uG1R3Ml5ZQImI5ZJmAJcC44CzImKhpKPz/NNJd3+dmwu3ADiykMQHgO9JWhO4BTiiqryamdnI\n",
       "VXmFQkT8Evhl3bTTC6//COwwwLrzgZdXmT8zMyuPn5TvbP3tzkDF+tudgYr1tzsDFetvdwYq1N/u\n",
       "DHQjdfMAW5Kil9tQzMyqUNWx01coZmZWCgcUMzMrhQOKmZmVwgHFzMxK4YBiZmalcEAxM7NSOKCY\n",
       "mVkpHFDMzKwUDihmZlYKBxQzMyuFA4qZmZXCAcXMzErhgGJmZqVwQDEzs1I4oJiZWSkcUMzMrBQO\n",
       "KGZmVopKA4qkaZIWSbpZ0rEN5m8o6aeS5ku6WtJOdfPHSZor6eIq82lmZiNXWUCRNA74GjAN2BE4\n",
       "RNKUusWOB66LiF2BdwGn1c0/BrgJ6N5xis3Mxogqr1CmAksi4raIeAa4ADiwbpkpwBUAEbEYmCxp\n",
       "EwBJWwKvB74FeNx4M7MON77CtLcA7iy8vwt4Rd0y84G3Ar+TNBXYCtgSeAD4KvBRYEKFeTSzQWiW\n",
       "zgC2B54EDo0T4pE2Z8k6WJVXKM1UU50MTJQ0F5gBzAVWSDoAuD8i5uKrE7N22h7YG9gfOL3NebEO\n",
       "V+UVyjJgUuH9JNJVynMi4nHgPbX3kpYCtwLTgTdJej2wNjBB0rcj4l31G5E0s/C2PyL6yyqAmfFk\n",
       "/n8NcHQ7M2LDJ6kP6Kt8OxHVtHdLGg8sBvYB7gbmAIdExMLCMhsAT0XE05KOAvaMiMPr0tkb+EhE\n",
       "vLHBNiIifAVjVhHN0kTSlcnRru7qHVUdOyu7QomI5ZJmAJcC44CzImKhpKPz/NNJd3+dKymABcCR\n",
       "AyVXVT7NbGA5iEwvO123zfSmyq5QRoOvUDqfDxzWiGapn9Q2A/DDOCFKD1rdosrfyEBpV3Xs9JPy\n",
       "VjU36lojbptZqcrfyKj+/qpslDeDCg8cvXz108tlyw6lybaZdpzBj3I+qgyuoxq4HVCsak0fOIah\n",
       "dvZF3kYp1SYdcjCvpGxVamW/tdg209K+aPHzayXtqj6TKn8jVaa9GgcUa5nEqj/YYOAv6sz4IrAp\n",
       "cL5mDrEsoHfuv5B179+MZ9d8mqX7vDx+/dnbB1z47t1ezOZz4YEdHmXhQR8rLc/373gA/3DTZgA8\n",
       "s9Y5wFvKyG9LeWihbK2mXVmeW9lvFe6LVvLRUtpVfd9a+I20lG6LaZciIrr2L2W//fnohT+IMyD6\n",
       "IWZDTBxi2X6IyH8/KGvZiID37v4IMwlmErx77zsGXXad+3/HQW8P1n641HxwxJ4PMpPgqH8K1r3n\n",
       "J6Xlt5U8tFC2ltOuKs+t7Lcq90Ur+WjlO1TV963K39MAy1d17Cw9wdH8c0AZav9UFiRm5+XmNJFu\n",
       "08tGBBy5x/3MJHjfTk/wmk9s1Y58sM59l3HQ24PnP/CnIZftgPy2nHZVeW5lv1W5LzogH52w7GDL\n",
       "O6A03FnV7JRO/uuQIDER4gdNfqGbXjYi4DWf2Ip3733HUAe6KvPR0rKt5Pf4dc7l33a+n0+ueRkz\n",
       "KW0fV5rnTtjHVe6LHl52sOWrOnb6OZQuI9FP8f79GLhhUGI26XbBa4D9YvC62ZVPRFddzzpGVfns\n",
       "RYfcSGBdouuelLfmtdjQ1sptgCvv8BgiSOT5HX8n0XB1yAG3yls4u+6usF5W1fetQ77HA/KDjRWQ\n",
       "OEOiX2J2PvMfSisPHx0K/JAhrjggBYkIpvuKA+iMByxXfnblHwj8oGBnqer71gnf4wH5CqUarZ4t\n",
       "Nn0w6IQriU4/SxpA2w+4VfWLlY3q8wY2pKq+b23/Hg/GAaVJFVZLQQtVUx2iG6tXevqAW3GwstZV\n",
       "9X3r6O+xG+Wb3lZLjeEtNXB32xm/ZmnVxv4Oz6+ZrcqN8u1XZbVUt3Uh0tFnSWbWHg4ozauyWqqq\n",
       "etFKApWrV8ysEd/l1aSK75iq6u6fjm7AM7Pe4jaUHtbK8K3d1o5jZsPnNpQu0ikH5yq7CDczq+cq\n",
       "r2p09MNHA3D1mJmNSOUBRdI0SYsk3Szp2AbzN5T0U0nzJV0taac8fZKkKyTdKGmBpA9WndcSdePB\n",
       "ucqnuM1sDKi0DUXSOGAxsC+wjHSAPSQiFhaW+RLwWER8RtIOwNcjYl9JLwReGBHzJK0H/Al4c926\n",
       "HdmG0krbhZnZaOvWNpSpwJKIuA1A0gXAgcDCwjJTgJMBImKxpMmSNomIe4F78/QnJC0ENq9bd0Ra\n",
       "Hv2sSVXeVtsp7TNmZvWqrvLaAriz8P6uPK1oPvBWAElTga2ALYsLSJoM7AZcXXL+urGtoxvzbGZj\n",
       "QNVXKM3Up50MnCZpLnADMBd4tjYzV3ddCBwTEU/UryxpZuFtf0T0t5C/bmzr6MY8m1kbSeoD+irf\n",
       "TsVtKHsAMyNiWn5/HLAiIr4wyDpLgZ1zNdcawC+AX0bEqQ2WHVE9YDcOKuX2GTMbqaraUKoOKONJ\n",
       "jfL7AHcDc1i9UX4D4KmIeFrSUcCeEXG4JAHnAQ9GxIcGSL8jG+XNzDpZVzbKR8RySTOAS4FxwFkR\n",
       "sVDS0Xn+6cCOwLmSAlgAHJlX3xN4J3B9rg4DOC4iLqkyz2ZmNjzuesXMbIzpyiuUXuLbdc3MBueu\n",
       "V5rn23XNzAbhgNI8365rZjYIB5Tmua8rM7NBuFHezGyMqerY6SsUMzMrhQOKmZmVwgHFzMxK4YBi\n",
       "ZmalcEAxM7NSOKCYmVkpHFDMzKwUDihmZlYKBxQzMyuFA4qZmZXCAcXMzErhgGJmZqVwQDEzs1I4\n",
       "oJiZWSkqDSiSpklaJOlmScc2mL+hpJ9Kmi/pakk7NbtuKfmbpTM0S/2apdmapYlVbMPMbKyoLKBI\n",
       "Ggd8DZgG7AgcImlK3WLHA9dFxK7Au4DTWli3DB7W18ysJFVeoUwFlkTEbRHxDHABcGDdMlOAKwAi\n",
       "YjEwWdI/NLluGTysr5lZSaoMKFsAdxbe35WnFc0H3gogaSqwFbBlk+uWwcP6mpmVZHyFaTcztvDJ\n",
       "wGmS5gI3AHOBZ5tcFwBJMwtv+yOiv+kMpiAyvdnlzcy6kaQ+oK/q7VQZUJYBkwrvJ5GuNJ4TEY8D\n",
       "76m9l7QUuAV4/lDrFtKYWU52zcx6Uz7R7q+9l3RCFdupssrrWmA7SZMlrUm6EriouICkDfI8JB0F\n",
       "XBkRTzSzrpmZdZbKrlAiYrmkGcClwDjgrIhYKOnoPP900h1c50oKYAFw5GDrVpVXMzMbOUU03VzR\n",
       "cSRFRKjd+TAz6yZVHTv9pLyZmZXCAcXMzErhgGJmZqVwQDEzs1I4oJiZWSmqfLCxLSTOIHX6+CRw\n",
       "aATuUsXMbBT04hWKexA2M2uDXgwo7kHYzKwNeu7BRomJpCuTo13dZWa2uqoebOy5gGJmZoPzk/Jm\n",
       "ZtbRHFDMzKwUDihmZlYKBxQzMyuFA4qZmZViwIAiaZqktzeYfpCk11abLTMz6zYD3jYs6Q/AmyPi\n",
       "/rrpmwAXR8Qeo5C/Qfm2YTOz1rXjtuG16oMJQEQ8AKxbdkbMzKy7DRZQ1pe0Rv3EPG3t6rJkZmbd\n",
       "aLCA8hPgDEnr1SZIWp/UrclPmkk8t8MsknSzpGMbzN9Y0iWS5klaIOnwwrzjJN0o6QZJ50taq+lS\n",
       "mZnZqBssoHwKuA+4TdJ1kq4DlgIPAJ8cKmFJ44CvAdOAHYFDJE2pW2wGMDciXgr0AadIGi9pMnAU\n",
       "sHtE7AyMAw5uoVxmZjbKBhwPJSKeAT4u6URgWyCAWyLiyYHWqTMVWBIRtwFIugA4EFhYWOYeYJf8\n",
       "egLwYEQsl/QY8AywjqRngXWAZU2XyszMRt2AAUXS20hBBEDACmCipHkR8XgTaW8B3Fl4fxfwirpl\n",
       "zgT+T9LdwPrAOwAi4iFJpwB3AE8Bl0bEr5rYppmZtclgIza+kZUBpWYjYFdJR0bEr4dIu5lujI8H\n",
       "5kVEn6RtgMsl7QJsCvwHMBl4FPiRpH+JiO/VJyBpZuFtf0T0N7FdM7MxQ1IfqVmhUoNVeR3eaLqk\n",
       "rYAfkaq0BrMMmFR4P4l0lVL0SuBzeXu3SFoKTAG2Bv4QEQ/mbf4kL7taQImImUPkw8xsTMsn2v21\n",
       "95JOqGI7LXe9EhG3A6vdTtzAtcB2kiZLWhOYDlxUt8wiYF8ASZsCOwC3AIuBPSQ9X5LyMje1mlcz\n",
       "Mxs9g1V5NSTpJcDfhlouN67PAC4l3aV1VkQslHR0nn868HngHEnzScHtYxHxEPCQpG+TgtIK4Drg\n",
       "jFbzamZmo2ewrlcubjB5Q2Bz4J0R8YcqM9YMd71iZta6qo6dg12hnFL3fgXwEKlh/p1A2wOKmZl1\n",
       "jsEa5ftrryXtDhxCuq13KfDjynNmZmZdZbDnUHYgBZHppKfjf0SqIusbnayZmVk3GawNZQXwC2BG\n",
       "RNyRpy2NiK1HMX+DchuKmVnr2tF9/VtJT6n/RtL/SNqH9MS8mZnZaga8QnlugdTb8IGk6q9XA98G\n",
       "fhoRl1WfvcH5CsXMrHVVHTuHDCh1mdgIOAg4OCJeU3ZmWuWAYmbWuo4IKJ3GAcXMrHXtaEMxMzNr\n",
       "mgOKmZmVwgHFzMxK4YBiZmalcEAxM7NSOKCYmVkpHFDMzKwUDihmZlYKBxQzMyuFA4qZmZXCAcXM\n",
       "zEpRaUCRNE3SIkk3Szq2wfyNJV0iaZ6kBZIOL8ybKOlCSQsl3SRpjyrzamZmI1NZ55CSxgGLgX2B\n",
       "ZcA1wCERsbCwzExgrYg4TtLGeflNI2K5pPOAKyPibEnjgXUj4tG6bbhzSDOzFnVj55BTgSURcVtE\n",
       "PANcQBpXpegeYEJ+PQF4MAeTDYC9IuJsgIhYXh9MzMyss1QZULYA7iy8vytPKzoT2EnS3cB84Jg8\n",
       "fWvgAUnnSLpO0pmS1qkwr2ZmNkLjK0y7mbq044F5EdEnaRvgckm75nztThrP/hpJpwIfBz5dn0Cu\n",
       "Nqvpj4j+EefczKyHSOoD+qreTpUBZRkwqfB+EukqpeiVwOcAIuIWSUuBHfJyd0XENXm5C0kBZTUR\n",
       "MbPEPJuZ9Zx8ot1fey/phCq2U2WV17XAdpImS1oTmA5cVLfMIlKjPZI2JQWTWyPiXuBOSdvn5fYF\n",
       "bqwwr2ZmNkKVXaHkxvUZwKXAOOCsiFgo6eg8/3Tg88A5kuaTgtvHIuKhnMQHgO/lYHQLcERVeTUz\n",
       "s5HzmPJmZmNMN942bGZmY4gDipmZlcIBxczMSuGAYmZmpXBAMTOzUjigmJlZKRxQzMysFA4oZmZW\n",
       "CgcUMzMrhQOKmZmVwgHFzMxK4YBiZmalcEAxM7NSOKCYmVkpHFDMzKwUDihmZlYKBxQzMyuFA4qZ\n",
       "mZXCAcXMzEpRaUCRNE3SIkk3Szq2wfyNJV0iaZ6kBZIOr5s/TtJcSRdXmU8zMxu5ygKKpHHA14Bp\n",
       "wI7AIZKm1C02A5gbES8F+oBTJI0vzD8GuAmIqvJpZmblqPIKZSqwJCJui4hngAuAA+uWuQeYkF9P\n",
       "AB6MiOUAkrYEXg98C1CF+TQzsxJUGVC2AO4svL8rTys6E9hJ0t3AfNIVSc1XgY8CKyrMo5mZlWT8\n",
       "0IsMWzPVVMcD8yKiT9I2wOWSdgX2Bu6PiLmS+gZLQNLMwtv+iOgfZn7NzHpSPo72Vb2dKgPKMmBS\n",
       "4f0k0lVK0SuBzwFExC2SlgIvydPfJOn1wNrABEnfjoh31W8kImZWkHczs56RT7T7a+8lnVDFdqqs\n",
       "8roW2E7SZElrAtOBi+qWWQTsCyBpU2AH4JaIOD4iJkXE1sDBwP81CiZmZtY5KrtCiYjlkmYAlwLj\n",
       "gLMiYqGko/P804HPA+dImk8Kbh+LiIcaJVdVPs3MrByK6N5jtaSICN8BZmbWgqqOnX5S3szMSuGA\n",
       "YmZmpXBAMTOzUlR523DbSOrehiEz6whun21dTwYU8JfBzIbPJ6XD4yovMzMrhQOKmZmVwgHFzMxK\n",
       "4YBiZmalcEDpEZK+KemTw1jvRZIelzSmbmKQNFvSYRWl/TpJP60i7dEkaYWkFw9z3Uq+V5L2krSo\n",
       "hHTWkrRQ0sZl5MsSB5Q2kHSbpH3KTDMi3hcRn21y268prHdHRKwfLfbBI+lwSc/mg8ajkq6X9Jbh\n",
       "5L0dIuL1EfGdipL/HHBS7c1AB2ZJb5D0O0kPS7pH0pmS1qsoT6NquN+revX7LiJ+GxEvKSF/fwfO\n",
       "Bj4+0rRsJQeU9gja1+FlUN4ImL+PiPWBiaThns+XtGFJaT9HUtd8TyW9HJgQEXOaWHwCcCKwGTCF\n",
       "NADdl1rY1rhhZbJidcN4l5JkyenVfB94t6Q1Kkp/zOmaH+pYkC/DT5W0LP99NXf9X5v/MUl3S7pL\n",
       "0r8Wz94knSvpM/n1xpJ+kc98H5T0GyXfAV4EXJyvLD6ShxdYUTtoS9pI0jl5+w8NUXUjgHwW+l1g\n",
       "LWCbQlm+LOl2SffmKrm1WyjLN3O11BNAn6TNJf1Y0v2SbpX0gUJaUyVdm6+U7pV0Sp6+tqTvSvpL\n",
       "3hdzJG2S5/VLOjK/lqRP5qu3+ySdJ2lCnlfbP+/KZXlA0vGD7JP9KYw7MZiI+H5EXBYRf4uIR0gj\n",
       "mO454M5OV4W/l/QVSX8BTpC05gj283P7oJD+bwfY9hskzc37+A4VxtMo7KP3SLod+JWkrWrfK0n/\n",
       "nL9vtb+/KY19VPvs/pg/n7sl/bfyAV7Sb/Im5uf13i6pT9KdhW1PyeV4WNICSW8szDtX0tfzb+Ex\n",
       "SVdp1audu4CHgX9u5vOyoY3JgCJxhkS/xGyJie1Ko4FPAFOBXfPfVOCTaXuaBnwI2AfYjtVHXyte\n",
       "9fwnafjljYF/AI6L5DDgDuCAXB3x5QZ5+A5pULMd87pfGSrTSmfKRwCPAIvz5JOBbXM5tiWdfX+6\n",
       "ybIAHAJ8JiLWA/4IXAzMBTbP6/2HpP3ysqcBX42IDYAXAz/I099NugrYEtgIOBr4W55X3F9H5GX7\n",
       "8vrrka64ivYEts/b/rSkgapd/rGwD1q1N7BgiGWmAreQPpvPA19g+Pu5lSvlJ4B35n38BuB9kg6s\n",
       "W+ZVpAHyXkfhqiIi/pi/b+sDGwJXAefn2ctJQ3+/gHRg3wd4f17vVXmZXfL6PypuLAeei4FLgE2A\n",
       "DwDfk7R9YbHpwMy83SXkAf0KFpL2nZUhIrr2j3xy3Oz0lfOjHyLy3w+Gt+3hpwEsBV7TYPoSYFrh\n",
       "/X7A0vz6bOBzhXnbACuAF+f35wAn5tezgJ8B2wy1bWByTud5pKqXZ4ENmijD4cAzpDO8p4EngT3z\n",
       "PJEOQC8uLP/PwK1NluVc4NzC/FcAt9dt/zjg7Pz6StJBY+O6ZY4Afg/s3CD/VwDvya9/DfxbYd72\n",
       "uUzPK+yfzQvzrwamD7BfLgPeWzftubINsj9fCzwEbDvEPr+98H6k+/m5fVBI/7fN5Bs4FfhK3Xdo\n",
       "cqPvVd163wQuGqSM/wH8ZKA8kILinfn1XsA9deufD5xQ+B6dUZi3P7CwbvnvAp9qkI8Y6jfQzX9V\n",
       "lW9MXqGQDn4A15DOWtuVRr3NgdsL7+/I0yAd7O8szKsfThlWnhV+iRScLpN0i6Rjm9z+JOChiHi0\n",
       "yeWviogNSWd/FwG17WwCrAP8KVdFPAz8knTF1ExZom7aVsDmtbRyeseRztIBjiQFgYW5WusNefp3\n",
       "SAO8XaBUhfcFNa7f34zV9/t4YNPCtHsLr58E1m2QDqQAO2GAeQ1J2gP4HvC2iFiSp+1VqCK6obB4\n",
       "cb+NdD+3ksdXSLoiVzk+QvrOv6BusTsbrFpM42jSVcyhhWnb5yqpeyQ9SrqCqE93IJs32ObtrPzN\n",
       "BHBfYd5TpKvPovVJn5mVYKwGlEOBHwL7RfBIG9OodzfpzK7mRcCy/Poe0gG/pvh6FRHxRER8JCK2\n",
       "Ad4EfFjSq2uzB9n+ncBGkjZoJdMR8VfgfcDekl4F/IX0490xIjbMfxMjonagbaYsxXzeQbpS27Dw\n",
       "NyEiDsjbXxIRh0bEJqQqoAslPT8ilkfEiRGxE/BK4ACg0VDSjfb7clY9GDXrelJwa4qk3YCfA4dH\n",
       "xBW16ZHuZlo//+1cWKW4X0a6n//KqoHxhYNk9XzSVe+WETER+B9WP34M+N2StBfpBoQDI+KJwqxv\n",
       "AjeRrsw2IFX7NntcuhuYJK1ya/JWrPzNNGMKML+F5W0QYzKgRPBIBNNHEghKSGNNpUbj2t940l0n\n",
       "n1RqVN+YVBf+3bz8D4EjJL1E0jrAp+rSe+5HJekASdvmH9pjpGqsFXn2feSG89XLFPeQznC/IWmi\n",
       "pDVygBhSRDwMnEFqr1lBamA+VSsbwbcotHk0XZZsDvB4bmB+vqRxkv5R0j/ltN9Z2w7wKOnAtkLS\n",
       "qyXtnNt4HidV0T3bIPvfBz6UG5fXI7VNXJDLMZCB7jyaTWoLqbdW3ef9PEn/SKr/nxERswfZVkMl\n",
       "7Od5wFvzPt2WdKU3kPWAhyPiaUlTSSdUTbW/SJqU83JY7QqsLt3HgSdzu9T76uYP+H0lVT0+CXws\n",
       "f1f7SCcNF9Q2PUS+tiC1rV3VTDlsaJUHFEnTJC2SdHOjqpd88LxE0rx8l8bhefqkfIl9Y57+warz\n",
       "Ospmk34Mtb9PA58FriWd5V6fX38WICIuAf6LVO/9Z1JDNcDf8/9iA+u2wOWkH+ofgK9HxJV53kmk\n",
       "oPWwpA8X1q05jHTgXUT6MQ+03xs16J4KvFrSLqTqryXAVbkq43LymXuLZakdOA8AXgrcCjxACl61\n",
       "M/HXAQskPQ58FTg40nMGmwI/IgWZm0h3XzV69uTsPP03Of0nSQ28xbI2Kv/qEyPmAo/mg27Rjaz6\n",
       "eR8BfJhUvXP2ANVbjbZZv92R7OevktqK7iO1wX23Lv3i6/cDJ0p6jBSYfsCqBttH+5CqJ3/coJwf\n",
       "IQWnx0if6QV1ac0Ezsvf14OK+yAingbeSGobeYB0I8VhEfHnwvbr81V8fyipre6ZBnm3Yah0TPl8\n",
       "ZrgY2Jd0GXoNcEhELCwsMxNYKyKOy2fli0kHgo2BF0bEvHzW+CfgzXXrRjTopn6g6b1E0hTgBmDN\n",
       "Ic6kO14vlQVA0muB90dERz3o2Wv7eSQkrUW6QtsrIv7SYH5PH0OqKl/VVyhTgSURcVs+C7gAqL/V\n",
       "8B5WnmlOAB7Mdd/3RsQ8SG0CpNv7NmcMk/QWpec7NiS1FVzUrQeGXipLvYi4vFOCSS/v55GIiL9H\n",
       "xJRGwcSGr+qAsgWr32WyRd0yZwI7Sbqb1Dh2TH0ikiYDu5HqTMey95KqJ5aQqqXq65u7SS+VpZN5\n",
       "P9uoqXrExmbq044H5kVEn6RtgMsl7RoRjwPk6q4LgWPq7g4ZcyJi/3bnoSy9VJZO5v1so6nqgLKM\n",
       "1W9brL8X/pXkp1cj4halLhl2AK5VehL2x8B3I+JnjTaQ22Bq+iOiv5ysm5n1hnwHXF/l26m4UX48\n",
       "qZF9H9I943NYvVH+K8CjETFL0qakxvddSA8bnUdqU/nQAOmP2UZ5M6tOrx9DqipfpQEFQNL+pNtJ\n",
       "xwFnRcRJSk/MEhGn5zu7ziE9TPY84KSIOF/S/yPdxnk9K6vOjsu3QtbSdkAxs9L1+jGkawNKlRxQ\n",
       "zKwKvX4M6dbbhs3MbIxwQOkR8hDALZGHAB6SPASwtcgBpQ3kIYDbLjpjCOAXSrpIqSfkFZJeVFF+\n",
       "Rt1wv1f16vddeAjgjuaA0h4eArgF6t0hgFeQ+nR72zC35SGAR8ZDAJesa36oY4E8BPBYGwL4/oj4\n",
       "H1InoEOShwD2EMAdbkwGFM3SGZqlfs3SbM3S8IYALiGNBjwE8EpjcQjgZngI4EweArjjjMmAQjow\n",
       "7E06mzy9jWnUO5Q0jO9fcqd1s0jdyQO8gzTk7cKIeAo4YaBESF2Sb0YakvXZiPh9MxuXtBkwjTQc\n",
       "7qO5k86GZ6zZHkqjBD5FGiXyjRFRa4g9CvhwRDySu8w5CTi4hbL8LCJq3a3vQhre97M5T0uBbxXS\n",
       "exrYTtLGEfFkobrpadKBarscUOfWuvSp8y/AKbkT07+SRoM8WKtWtc3KHQpeT+pzbqCD0ETSsAFV\n",
       "uTsivp47ePw7I9/PTYmIKyPixvz6BlJHr/XjvsyMiKdy+8RA/ht4LCI+kdO6LiLmRMSKiLid1IV9\n",
       "o/FkGtkDWDciTs7fiyuAX5BORmp+EhHXRsSzpFExX1qXxuOkz8xKMFYDiocAbsxDACejOgRwI/IQ\n",
       "wM3wEMAdZqwGlJXD954QIx8CePhp1PMQwIVkC697dgjggYSHAG6GhwDuMGMyoMQJ8UicENNHEghK\n",
       "SMNDAI/xIYBz3tcmtVkBrK1Cg/pQStjPHgLYQwCXakwGlA7hIYDH9hDAh+fpT5KCfpD2+V8bpVvY\n",
       "pocA9hDAHct9eXUp9dBwrr1UFvAQwN1AHgK4kvI5oHQRpSfRZ5MaYs8DlkfEW9ubq+HppbJ0Mu/n\n",
       "4enVY0gLRacRAAAJjElEQVRNVeVzlVd36aXhXHupLJ3M+9lGTdUjNo46zdIZ+f9s4NAS78Bqu+ih\n",
       "4Vx7qSydzPvZRlMvXqHUbtks84FDMzMbQi8GlCoeODQzsyH0YkCpPYVb5gOHZmY2hJ69y6sd+TGz\n",
       "3uG7vFpXaaO8Um+npwLjgG9FxBfq5m9MepjqhTkvX46Ic5tZdzC9/EUwM+tUlVV55e4uvkbqvXZH\n",
       "4JD8YFXRDGBuRLyU1LX2KZLGN7luz8tdSfQsl6+79XL5erlsVaqyDWUqsCR3Cf4MqUuF+vET7mFl\n",
       "9xkTgAcjYnmT644Ffe3OQMX62p2BivW1OwMV62t3BirU1+4MdKMqA8oWrN519hZ1y5wJ7CTpblKP\n",
       "n8e0sK6ZmXWQKgNKMw3jxwPzImJzUsd/X5e0foV5MjOzilTZKL+M1cdiqB/g55XkITkj4halcaZ3\n",
       "yMsNtS7Q+3d0qTB2dy9y+bpbL5evl8tWlSoDyrWkYVknkwbCmc6qQ3NC6q57X+D3kjYlBZNbSV1Z\n",
       "D7Wu7+YyM+sglQWUiFguaQZpCNZxwFkRsTAPA0pEnE4ayOgcSfNJ1W8fi4iHABqtW1Vezcxs5Lr6\n",
       "wUYzM+scXdv1iqRpkhZJulnSse3OT7Mk3SbpeklzJc3J0zaSdLmkP0u6TNLEwvLH5TIuKgztiqSX\n",
       "SbohzzutHWXJ+Thb0n2FEfhKLY+ktST9IE+/StJWo1e6Acs3U9Jd+TOcK2n/wryuKZ+kSZKukHSj\n",
       "pAWSPpin98TnN0j5euXzW1vS1ZLmSbpJ0kl5evs+v4jouj9SNdgSYDKwBmnktSntzleTeV8KbFQ3\n",
       "7Yuk6j5IQ7qenF/vmMu2Ri7rElZeVc4BpubXs4FpbSrPXsBuwA1VlIc09Ow38uvppLHe212+E4AP\n",
       "N1i2q8pH6qHipfn1esBiYEqvfH6DlK8nPr+8zXXy//HAVcD/a+fn161XKN3+4GP9zQRvIo2mR/7/\n",
       "5vz6QOD7EfFMRNxG+gK8QtJmwPoRMScv9+3COqMqIn4LPFw3uczyFNP6MWl88lEzQPlg9c8Quqx8\n",
       "EXFvRMzLr58AFpKe9+qJz2+Q8kEPfH4AEVHrXX1N0on2w7Tx8+vWgNLNDz4G8CtJ10o6Kk/bNCLu\n",
       "y6/vAzbNrzdn1dula+Wsn76Mzip/meV57rOO1IvCo5I2qijfrfiApPmSzipUKXRt+ZTuqNwNuJoe\n",
       "/PwK5bsqT+qJz0/S8yTNI31OV0TEjbTx8+vWgNLNdxLsGRG7kQYA+3dJexVnRrq27ObyraLXypN9\n",
       "E9ia9DDuPcAp7c3OyEhaj3T2eUxEPF6c1wufXy7fhaTyPUEPfX4RsSJSX4hbAq+S9Oq6+aP6+XVr\n",
       "QGnmocmOFBH35P8PAD8lVd/dJ+mFAPny8/68eH05tySVc1l+XZy+rNqct6SM8txVWOdFOa3xwAaR\n",
       "by1vl4i4PzLgW6TPELqwfJLWIAWT70TEz/Lknvn8CuX7bq18vfT51UTEo8D/Ai+jjZ9ftwaU5x6a\n",
       "lLQmqbHoojbnaUiS1lHuWkbSusB+wA2kvL87L/ZuoPbDvgg4WNKakrYGtgPmRMS9wGOSXiFJwGGF\n",
       "dTpBGeX5eYO0DgJ+PRoFGEz+kda8hfQZQpeVL+flLOCmiDi1MKsnPr+BytdDn9/Gteo6Sc8HXgvM\n",
       "pZ2f32jekVDmH6nKaDGpYem4duenyTxvTbrLYh6woJZvYCPgV8CfgcuAiYV1js9lXAS8rjD9ZaQf\n",
       "whLgv9pYpu+TejN4mlTXekSZ5QHWAn4I3Eyq/57c5vK9h9RoeT2pQ9Ofkeqsu658pDuCVuTv49z8\n",
       "N61XPr8Byrd/D31+OwPX5fJdD3w0T2/b5+cHG83MrBTdWuVlZmYdxgHFzMxK4YBiZmalcEAxM7NS\n",
       "OKCYmVkpHFDMzKwUDijWsSStkPTlwvuPqKRhWSWdK+ltZaQ1xHbenrsWX+2BMElfUupW/QvDSHdX\n",
       "FbpdN+sEDijWyZ4G3iLpBfl9mQ9NDTut3AVFs44E/jUiGvXSehSwc0QMZzyf3YDXt7KCsmFsy6wp\n",
       "DijWyZ4BzgA+VD+j/gpD0hP5f5+kKyX9TNItkk6WdJikOUoDm724kMy+kq6RtFjSG/L64/KVw5zc\n",
       "G+17C+n+VtLPgRsb5OeQnP4Nkk7O0z4N7AmcLemLdctfRBqj4zpJ75C0iaQL83bnSHplXm6qpD9I\n",
       "uk7S7yVtn7sbOhGYrjRA1DuUBo36z0L6CyS9KHdPtFjSeaQnoSdJ+mihfDPz8utK+l+lwZpukPSO\n",
       "Fj8rs+rGlDcryTeA6+sPyKx+hVF8vwvwEtLYEEuBMyNiqtKIfR8gBSgBW0XEyyVtC1yR/78beCQv\n",
       "vxbwO0mX5XR3A3aKiNuLG5a0OXAysDvwCHCZpAMj4kSl3l//MyKuWyWzEW+S9HiknqeRdD7w1Yj4\n",
       "vaQXAZeQBkRaCOwVEc9K2hf4fEQcJOlTwMsiojYKYX1VYHF/bAscFhFzlEbp2zaX73nAz5V6vN4E\n",
       "WBYRtcA6AbMWOaBYR4uIxyV9G/gg8FSTq10TeTwISUuAS/P0BUCte+8g9VFERCyRdCspCO0H7Czp\n",
       "oLzcBNIBeTmpI71Vgkn2ctJYFA/mbX4PeBUrO9hrppppX2BKoUZqfUnrABOBb+dgF6z8zarJdAFu\n",
       "j5WDJ+0H7Cdpbn6/bi7f74BT8tXVLyLid02mbfYcBxTrBqeSOsE7pzBtObnKNp9pr1mY9/fC6xWF\n",
       "9ysY/DtfO6ufERGXF2dI6gP+Osh6xYO7WPUKoZn2GgGviIin67b7DeDXEfEWpfG8+wdY/7n9ka1d\n",
       "eF2f75Mi4ozVMiDtBrwB+KykX0fEZ5rIt9lz3IZiHS8iHiZdTRzJyoPzbaQeUiENU7pGi8kKeHtu\n",
       "p94GeDGpB9ZLgffXGt5zm8U6Q6R1DbC3pBdIGgccDFzZYn4uI12Fkbe7a345gdTbMaSenGseA9Yv\n",
       "vL+NVOWGpN1JPVs3cinwHqXhE5C0RW6/2Qz4W0R8D/hyLS2zVjigWCcrntmfAmxceH8m6SA+D9gD\n",
       "eGKA9erTi8LrO4A5wGzg6Hx18C3gJlJj+Q2k0f3G1627aqJp0LSPA1eQuhK/NiIubrF8HwT+KTeU\n",
       "3wgcnad/EThJ0nWkMcNr61wB7Jgb5d9OGkRqI0kLgH8nDe2w2nbyldf5wB8lXU8K1OuTukK/OleF\n",
       "fQrw1Ym1zN3Xm5lZKXyFYmZmpXBAMTOzUjigmJlZKRxQzMysFA4oZmZWCgcUMzMrhQOKmZmVwgHF\n",
       "zMxK8f8BjPRrq1X0SokAAAAASUVORK5CYII=\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1125a9f50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "auc_map = {}\n",
    "for i in xrange(500,30000,1000):\n",
    "    count_vect_top_temp = CountVectorizer(max_features=i)\n",
    "    matrix_counts_top_temp = count_vect_top_temp.fit_transform(all_texts).toarray()\n",
    "    X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_top_temp, target)\n",
    "    logreg_count_top_temp_predicted, _ = logistic_reg(X_train, X_test, Y_train, penalty='l1', C=1024)\n",
    "    auc_map[i] = get_auc(Y_test, logreg_count_top_temp_predicted)\n",
    "    \n",
    "auc_map2 = {}\n",
    "for i in xrange(500,30000,1000):\n",
    "    count_vect_top_temp = CountVectorizer(max_features=i)\n",
    "    matrix_counts_top_temp = count_vect_top_temp.fit_transform(all_texts).toarray()\n",
    "    X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_top_temp, target)\n",
    "    logreg_count_top_temp_predicted, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "    auc_map2[i] = get_auc(Y_test, logreg_count_top_temp_predicted)\n",
    "    \n",
    "plot.plot(auc_map2.keys(), auc_map2.values(), '.', label='Logistic Regression (L2-regularization)')\n",
    "plot.plot(auc_map.keys(), auc_map.values(), '.', label='Logistic Regression (L1-regularization)')\n",
    "plot.xlabel('Number of features'); plot.ylabel('AUC'); \n",
    "plot.legend(loc=0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "roc_curve() takes at least 3 arguments (3 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-62-fb8283220b43>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0mlogreg_counts_l2_predicted_tfidf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogistic_reg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.02915728098552524\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0mroc_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogreg_counts_l2_predicted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'freq'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinreg_bincounts_l2_predicted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'binary'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogreg_logcounts_l2_predicted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'logfreq'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: roc_curve() takes at least 3 arguments (3 given)"
     ]
    }
   ],
   "source": [
    "matrix_counts = matrix_count\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts, target)\n",
    "logreg_counts_l2_predicted, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "def bin_f(x):\n",
    "    return 1 if x > 0 else 0\n",
    "n_vec_bin = numpy.vectorize(bin_f)\n",
    "matrix_counts_bin = n_vec_bin(matrix_counts)\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_bin, target)\n",
    "binreg_bincounts_l2_predicted, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "import math\n",
    "\n",
    "def log_f(x):\n",
    "    return math.log(x+1)\n",
    "n_vec_log = numpy.vectorize(log_f)\n",
    "matrix_counts_log = n_vec_log(matrix_counts)\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_log, target)\n",
    "logreg_logcounts_l2_predicted, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "tfidf_vec = TfidfVectorizer()\n",
    "matrix_tfidf = tfidf_vec.fit_transform(all_texts).toarray()\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_tfidf, target)\n",
    "logreg_counts_l2_predicted_tfidf, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression ( freq ) AUC =  0.933953033268\n",
      "Logistic Regression ( binary ) AUC =  0.945350026686\n",
      "Logistic Regression ( logfreq ) AUC =  0.943871197296\n",
      "Logistic Regression ( tf-idf ) AUC =  0.876278687066\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAdgAAAEPCAYAAADh4tUOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXHWZ5/HPlwAGBJZqoo5cAgiI4A4IrtFRcRphNCAL\n",
       "6ggMKLOIs7JqHF8royDDks56gxUVFRczyMVFJeKICiwCKrYiV1EE1OCAEA3hJumOA8olMc/8cU4l\n",
       "p6u7qk9116+u3/fr1dDnnF+devokqad/d0UEZmZm1lqbdDoAMzOzfuQEa2ZmloATrJmZWQJOsGZm\n",
       "Zgk4wZqZmSXgBGtmZpZA0gQr6QJJj0i6q0GZz0q6R9IdkvZLGY+ZmVm7pK7BXggsrHdR0qHA7hGx\n",
       "B/BO4NzE8ZiZmbVF0gQbEdcD4w2KHA58KS97C7CtpOeljMnMzKwdOt0HuwOwsnD8ALBjh2IxMzNr\n",
       "mU4nWADVHHvtRjMz63mbdvj9VwE7FY53zM9NIMlJ18xsBiKithJjbdLpBHs5sAhYJukVwJqIeGSq\n",
       "gv5LkpE0EhEjnY6jG3Tzs9ASjQGVJl82HotjaPqbK6j599DNz6IZEjN5bjVG8q8Z+vaPYZt1swth\n",
       "o/EYHp7+zzQRV046K2mClXQJ8NfAPEkrgcXAZgARsTQirpJ0qKR7gT8Cb08Zj9lsNZE4x2NxCDWV\n",
       "aCuMlPpA3DBwcGNCWozE4pLv083GIyZ1GzVFWjISMTIy6fzoaPk/uw4mResfSRNsRBxTosyilDHY\n",
       "4JphLXI6WeKc8EZTvk81WY7X1jbLmKYmV0EbxiqMR6B6SWVQ1Umm4zE87JYwa5tONxFb80Y7HUAX\n",
       "GZ3qZCGxTk6GrbQxsc4oida/LWMATdbkRlv1/ik1UYucuU9/GpxMrQuoFzZclxTug7WqYs109Rkw\n",
       "9FTHQhknYqg1/YYT7xtBVzZRtiBBuvm1jfzZ2VmuwVpHzaQZd+wMorIxqY4TJQYGtVAhoVabamfd\n",
       "b9gL8uSKa4a9z4OfWqveLzFOsNZplSmbcRsPDlrT7qRaNcPm255WqLW69tlHXLNtjUa/rDjBWnfJ\n",
       "E+s428ZQ/VU2i4N82q1rm2+b0WRTr/szzWbAfbDWcrXNvk32k1b7NWOQaomtMIOk2fO/KFhj9T47\n",
       "/ZnaOo2epROstdzYFopiQs1ro83++fVFTTGlKRKqk6ZN4ASbXqNn6SZim7l6/aRzgQgVBgOtcbJs\n",
       "LQ84sn4laU/ga8ALgFMj4pwOhzRjTrDWnIlJddL8T51SWQ/SII2ubbdCcvUvLdaPPgh8PyJe0ulA\n",
       "ZssJ1uqbooZaM/iowilDwRbFwUjbRpwxJs5oV5D9pWQ/qpuCrZ/tDNw41QVJm0TE+jbHM2Pug7VG\n",
       "U2I2zDEtNPdO6BvVEkXS1ZL60DRJ1MnTWqbX+mAlXQe8BlgLrCPbEObfyZLua4DDgbuBzwEHAE8A\n",
       "n46Iz+Wv3wI4Ny/3EHAR8I8RsROJuA/WplMpsdRfxc29s+e+U7P6IuK1kn4AXBwRF0i6CDgGOCQi\n",
       "bsoT6I+BbwJHk213+j1Jv46Ia8k2lNmVrP92K+BqOrjHuBPsICuupTvpEmOcPFTZ0Pw7Aloy5V/U\n",
       "upNVB8EMlg50DdW6nlo0z7xFv5R/KyJuyr/fB5gXER/Jj++X9EXg74BrgSOBd0XEGmCNpM8Ap7cg\n",
       "hhlxgh0wxTmqAWgEgApLalYjyc6X25+0z3gRBht0XdRaFcCqwvHOwPaSir/YzwF+lH+/PbCycO13\n",
       "acNrzAm2x23oGz15iGpts9HCDjW/lg5kAoVS/aDd8gFjNuiKH1u/A+6PiBfWKfsQMB9Ynh/PTxnY\n",
       "dJxge5ROqaxnizXKa5pQTJYjilZun9arnETNel7tv9FbgcclfZBsoNMzwF7A3Ii4DbgU+JCkW8j6\n",
       "YN+L+2CtjA1JFYBtJ47elcbyDb7B/aLFxemdRM16V1BIkBGxXtJhwCeB+4BnkY0qPi0vsgT4AnA/\n",
       "WdPyRcD72hjvBJ6m08Um7DN68hAQEWeMb1JTqLjp98A093qqi9n0em2aTqtJGiYbkexpOpaRGFvN\n",
       "UCWKFdEzs0ucOWlrpEmrKfUz107NrFc4wXanyhDjFBOnF3TYoOLEamZNcB+swbgq6yusUQBjc2G7\n",
       "iVNnBrZftaY5eGCfg5k1JyJG6eBIYifYDqszL3U8FsdQ9/eOt41rrWbWc5xg22iqjchj4nxVz0ud\n",
       "mmutZtZznGDbq7KhHzUb/QsRQ/loYQZpz9Q6m4W7lmpmfcPTdFql/o409RR3qokuWpqsJUosN+ip\n",
       "NGaJDfo0nXbwNJ32qGiEcWprZdM0+ea1135sAnW/qZkNNCfYsqavoY5TbAIur++2gctrr/34S4OZ\n",
       "JSRpBfCOiPh+zfkDgPMi4kUdCWyGnGCnM3GlpOK81NqEW3/bt+kTc09x86+ZJTJhacQNJyOuB3oq\n",
       "uYITbBn1NiMvW1vtuxoqbv41sz4gSQCRaDDSJtMXGWBZ7bXnaphmZj1sgaRfShqTdIGkZ0kalrRh\n",
       "n1dJKySdJOkOSWskLZP0rPzatpKulPRofo8rJO1QeO2opI9IugH4I3CSpNuKAUh6v6RvzfYHcQ22\n",
       "nnwajUZg0mbkmYaJt9A03NUJusnNxau6+mcys54l4FjgdcCfgCvIdsr5Xk25AI4EXg88DdwAHA8s\n",
       "Jas4ng+8hSzHXQCcA7yp8Pq3AYcAv87LfEjSiyLi7vz6ccD/nu0P4wRbX9Y0XGcNYIkxjTRc43K8\n",
       "R5qG3dxrZhNo6kpF02Yw6DOAcyJiFYCkj5Lt+1qbYAE+GxEP5+WuAF4CEBFjwDerhSR9DLiu5j0u\n",
       "iojqpuzPSLqULOmeJunFwM7AlU3GPokTbNHEkcLj+UCm8Y2XJwxY6pUEOonX9jWzRjq8scjKwve/\n",
       "A7avU+7hwvdPVstJ2hL4NFnttvo5t5XyCatTvAfAl4CvktWWjwO+FhFrZ/wT5JxgqzaurCTYMEqY\n",
       "6jzWwmpLPZNUGzT/etUkM+tW82u+f7DJ158EvBBYEBGPSnoJ8DOy5udqgp1QQ4+ImyU9I+k1wDH5\n",
       "16w5wW5UO1p4wyjhHl7K0M2/ZtZLBLxH0pVktdJ/BpY1eY+t8tf+QdIQsLjO+9S6mKyv9pmIuLHJ\n",
       "95ySRxEXaImi+kXedNrDydXMrNcE8BXgWuA3wD3AR5hY+6z3uur1s4EtgMeAG4HvTPHaqe51MfBi\n",
       "4MszjH2SpGsRS1pI9sPOAb4YEWfWXJ9H9sP8BVlt+qyIuGiK+yRZN7O6WMTqM0AQlSdj0i8cvbxO\n",
       "sEZHwzVYs8HltYjLk7QF8AiwX0T8ponX1X2WyWqwkuaQVbcXAnsDx0jaq6bYIuD2iHgJMAx8UlI7\n",
       "m60rsTg09BRUk6vEmERUv/AgIDOzQfAu4NZmkut0UiazBcC9EbECQNIy4AhgeaHMQ8A++ffbAKsj\n",
       "Yl3CmMro2ZWXptoCrlOxmJn1inwN5ADe2Mr7pkywOzBxKPQDwMtrypwHXCfpQWBr4KiE8UxQOwWn\n",
       "lxUSq0cHm5k1KSJ2SXHflAm2TOfuqcDPI2JY0m7AdyXtGxGP1xaUNFI4HI2I0VnGN5Odb7pOnlxx\n",
       "YjUzScNk3W3WBVIm2FXAToXjnchqsUWvBD4KEBG/kXQ/sCdwW005ImIkTZiZblzasOQyht65xswA\n",
       "yCseo9VjSVNNUbE2SZlgbwP2kLQL2UTho5k8efdu4GDgBknPI0uu9yWMaRKJsQ0zj7uv79XzWM3M\n",
       "elSyBBsR6yQtAq4hm6ZzfkQsl3Rifn0p8DHgQkl3kI1o/mC+jmRyq88ARlSdpDTeTfNci32qnY7F\n",
       "zMxmJuk82FZp2ZytwlrDY3Nhu6eiq2qtNYOVuibhm1lv8jzY9Bo9y8FZKnHj9nPjQIUntw3OZE2H\n",
       "o6rlJmEzG1j5dJl3RMT3Z3GPLYBLgQOAayLi6BaF17TBSbA128/lKzR1tJboeatmZhMUlzycqbcA\n",
       "zwWGImL97EOauUFKsBTXGO40T68xM0tiZ+Df6iVXSXMi4s/tCGSgFvuPxSFGgi5ZArHiflYzs8kk\n",
       "bS7pbEmr8q9PS9q8cP2Dkh6U9ICkf5C0XtJukpYA/ws4WtLjkk6QdLykGyR9StJjwOL8/mdJ+q2k\n",
       "hyWdK2lu4f4fKNz/hPz+L2j25xioBJurRKBONg/ntddOJ3gzs24kso3PFwD75l8L8nPVTWT+J3AQ\n",
       "sAcbF9aIiFhMNjtlWURsHREX5NcWkO3O89z8+pnA7vm9dydbefD0wv1PIptC+sL8/zNqth6oJuJ2\n",
       "m2bDc9dezaw7Sa2ZXjLzkcrHAosi4rEsHC0BlpIlwaOACyJieX5tcV6+Skze7/XBiPh8Xv5p4L8D\n",
       "+0TEmvzcx8m2yTu1cP9fFe7/dzP5Ifo/weZTc8bmiu0SNA1Ps9qS1wY2s97T+Sk82wO/LRz/Lj8H\n",
       "8Hzg1sK12hUCp1JcF/85wJbAT6UNP6bY2KL7fOAnNe89I/2fYKGiEWBkfao5r55aY2bWWg8Cu7Bx\n",
       "97X5ZMvvQrYLW+0yvNMp1sgfA54E9o6Ih6Yo+1D+flXzpyhTyiD2wZqZWXe7BDhN0jxJ88iahr+c\n",
       "X7sUeLukF0nakmxQU2n56OLzgLMlPQdA0g6SXle4//GS9srvP+P1nPs7wUpj43PTjBjW6OiYRke7\n",
       "YTSymVk/CeAjZOvZ35l/3ZafIyKuBj4L/AD4N+Cm/HVPF14fNfer7VM+GbgXuFnSH4Dvkg1oqt7/\n",
       "bOC6/P4zX/Sir5dKlEIjjMfiGMoXlmhZU65GR8NNw2bWzQZhqURJewF3AZunWlhC0npg94iYtBnN\n",
       "QC6VqCUaCyBPri2bFuOF+M3MOkvSm4CryAYrnQlc3ulVm6bStwl29RkTkmClhbVXD2oyM+usdwIX\n",
       "An8m2//23Ynfz/NggY075swFIlo619QLRJiZdV5EHNLm95szk9f1X4LNF/XfbolS9C679mpmZqX0\n",
       "1yjifNRwNy3qb2Zmg6l/arD5fq9Dp6BY3B+j48zMrHf1Rw02T661fa4SY63aOcf9r2Zm1ox+qcFW\n",
       "6qyd6dHDZmbWEf1Rg81piZLUMl17NTPrDElvkrQy399135pr8/PzU1Z+JI1IurjMvVLoqwQLVGJx\n",
       "kqk5eHs5M7P0JK2Q9NrCqbOAd+f7u95RLBsRv8vP15s0Unu+7r1S6Jcm4klauHqTm4bNzNonyPdz\n",
       "zWum84FfzfamrbxXWf1Wgy2qRDCrWqebhs3M2idvzp0PXCHpT8ATwBzgDkn3TFF+F0nrJW2SH+8q\n",
       "6YeS/l3StcC8/PzmwOON7pVCPyfYWXHTsJlZe0XEcWQbnB8WEVtGxLPzS/tExB4lbvFVss3StwM+\n",
       "DPy37LbxTERs1eS9Zq1vmoirA5zypuFWLMbvpmEzG0j5Vpyz1s7PUEnzgf8CvDYi1gLXS7oCWreL\n",
       "WrN6PsFWd82BfOeckdltS+fdcsxs0HVr5ULSE2zc33XvmsvbA+MR8WTh3G+BndoU3iQ9n2DJkiEt\n",
       "HD3smquZWefUrT0XmnmBrA+2cPgQUJG0ZUT8KT+3M9Cxbez6pg+2Fas2eVCTmVnHPQLs1uyLIuK3\n",
       "wG3AEkmbSXo1cBgz3GquFfomwZKv2jTTkcMe1GRm1hU+DpwmaVzSSUyfIIvXjwVeDowBpwNfalA2\n",
       "uX5oIq7yoCYzsx4XEZcDlxdOfbJB2RVkU2+qx/cDr2lQfkb7us5U39RgZzPn1U3DZmbWaj2dYFu4\n",
       "9nDFTcNmZtZKpRKspC0l7Zk6mBmY9drDrr2amVkK0yZYSYcDtwPX5Mf7Sbq88avaSBoby2bqzJRr\n",
       "r2Zm1nJlarAjZKOyxgEi4nbgBQljalZlO8Zm9ELXXs3MLJUyCXZtRKypOVdq4q6khZLulnSPpJPr\n",
       "lBmWdLukX0gaLXPfKcw0Sbr2amZmSZSZpvNLSW8FNpW0B/CPwI3TvUjSHOAc4GBgFfATSZdHxPJC\n",
       "mW2BzwOvj4gHJM0rG7iWaIwntw1YM+O5r2Zmg0pSxxZgGBRlEux7gX8GngYuIeuL/XCJ1y0A7s3n\n",
       "KSFpGXAEsLxQ5ljgGxHxAEBEPFY6cqisPlPgJl4zs6ZEhOf8t0GZBLtnRJwKnNrkvXcAVhaOHyDr\n",
       "yy3aA9hM0g+ArYHPRMTFjW6aT82pAONDjFeImY0idv+rmZmlVCbBfkrSXwBfB74WEb8oee8yzQ+b\n",
       "AfsDBwFbAjdJujkiGm2GW4nF+W9fI7Nq4vDKTWZmlsy0CTYihiU9HzgKWCppG+DSiJiumXgVE7cJ\n",
       "2omsFlu0Engs317oSUk/AvYFptq5foQDOBl4SiMaDrhsjErTna/ejs7M+pWkYWC4w2FYThHlK4GS\n",
       "/hI4GTg6IjabpuymwK/JaqcPArcCx9QMcnoR2UCo1wPPAm7J7/2rmntFREhLFBtqr1KIoNm9XzU6\n",
       "Gq65mtkgqH52djqOQTVtDVbS3mS117cAq4GvAe+f7nURsU7SIrJBUXOA8yNiuaQT8+tLI+JuSVcD\n",
       "d5JN/TmvNrmamZn1ojJ9sBcAy8im0qxq5uYR8R3gOzXnltYcnwWc1cx9zczMul2ZPthXtCMQMzOz\n",
       "flI3wUr6ekQcKemuKS5HROyTMC4zM7Oe1qgG+778/4fBpIFEPbkCiOe+mplZu9RdizgiHsy/fXdE\n",
       "rCh+Ae9uS3St57WHzcysLcos9v+6Kc4d2upAzMzM+kmjPth3kdVUd6vph90auCF1YGZmZr2sUR/s\n",
       "V8mm2JxBtrhEtR/28YhYnTqwVnP/q5mZtVOjBBsRsULSe6gZ1CRpKCJmtst553jtYTMza5tGCfYS\n",
       "4A3AT5l61PCuSSIyMzPrA3UTbES8If//Lm2LxszMrE9MO4pY0qskbZV/f5ykT0naOX1oZmZmvavM\n",
       "NJ0vAH+StC/ZIv/3Af8vaVRmZmY9rkyCXRcR64E3Ap+PiHPIpuqYmZlZHWV203lc0qnA24ADJM0B\n",
       "Gu4Fa2ZmNujK1GCPBp4GToiIh4EdgE8kjcrMzKzHTZtgI+Ih4CvAtpIOA56KCPfBmpmZNVBmFPFR\n",
       "wC3AkcBRwK2SjkwdmJmZWS8r0wd7GvCyiHgUQNJzgO8DX08ZmJmZWS8r0wcr4PeF49VM3h/WzMzM\n",
       "CsrUYK8GrpH0VbLEejTZJgBmZmZWx7QJNiI+IOnNwKvzU0sj4ptpwzIzM+ttjfaDfSHZdJzdgTuB\n",
       "D0TEA+0KzMzMrJc16oO9ALgS+FvgZ8Bn2xKRmZlZH2jURLxVRJyXf3+3pNvbEVAK3mzdzMzarVGC\n",
       "nStp//x7AVvkxyLbjP1nyaOb3rRJM0+uxPDwUPpwzMzMMo0S7MPAJxscH5gkoulIY0BljAoRlEma\n",
       "lRge9rQiMzNrq0Ybrg+3MY5mVIjQdiKi05GYmZnVUWahia4jEbhP1czMuliZhSa6gpaoOlCpEuGV\n",
       "pMzMrLv1Ug22EoujqYFKHj1sZmadUmY3nU0kHSfp9Px4vqQF6UObHY8eNjOzTipTg/2/wF8Bx+bH\n",
       "T+Tn2k5irIniFSdXMzPrlDJ9sC+PiP2qC01ExJikzRLHNaXVDFVwk6+ZmfWAMjXYZyTNqR7k+8Gu\n",
       "TxdSfUOMQzTXD2tmZtYJZWqwnwO+CTxX0seAt5Btwt418v7WSs1p13TNzKxjymxX92VJPwUOyk8d\n",
       "ERHL04ZVXmEwk6fumJlZ1ygzing+8Efgivzrj/m5aUlaKOluSfdIOrlBuZdJWpfvOzul1WfA2KRK\n",
       "KuDBTGZm1oXKNBFfBVRXJZwL7Ar8Gnhxoxfl/bbnAAcDq4CfSLq8tvablzsTuBrqLyAx9BSIMbw8\n",
       "opmZ9YIyTcT/uXic76jznhL3XgDcGxEr8tctA44AapuX3wv8K/CyEvecwAtJmJlZt2p6Jad8m7qX\n",
       "lyi6A7CycPxAfm4DSTuQJd1zq7dvMhw3D5uZWVeatgYr6aTC4SbA/mRNvtMpkyzPBk6JiJAkGjQR\n",
       "m5mZ9ZIyfbBbFb5fB1wJfKPE61YBOxWOdyKrxRa9FFiW5VbmAYdIWhsRl9febCT/r7RkBBiNiNES\n",
       "MZiZDQxJw8Bwh8OwnKLBrqr5AKT/ExEn1S1U/7Wbkg2GOgh4ELgVOKbeFB9JFwJXRMRlU1yLAERQ\n",
       "3ElHo6Ph6TlmZlOTFBHhz8gOqVuDlbRpRKyT9Crlf0rN3Dh/7SLgGmAOcH5ELJd0Yn596awiNzMz\n",
       "62J1a7CSfhYR+0v6ArA98HXgT/nlmKqmmYprsGZmzXMNtrMa9cFW/1DmAquB19Zcb1uCNTMz6zWN\n",
       "EuxzJL0fuKtdwZiZmfWLRgl2DrB1uwJplheZMDOzbtYowT4cEUvaFknzKu5/NTOzbtX0Sk6dMja3\n",
       "0xGYmZmV1yjBHty2KErY7pROR2BmZlZe3QQbEavbGYiZmVk/6ZkmYjMzs17iBGtmZpaAE6yZmVkC\n",
       "TrBmZmYJ9FqC9cISZmbWE3oqwUYw1OkYzMzMyuipBGtmZtYrGi2V2JXyNYgruLnYzMy6WM8lWLwG\n",
       "sZmZ9QA3EZuZmSXgBGtmZpZA7yTYV34bjY4G7ns1M7Me0Dt9sJttg/tezcysV/RODdbMzKyHOMGa\n",
       "mZkl4ARrZmaWgBOsmZlZAk6wZmZmCTjBmpmZJeAEa2ZmloATrJmZWQJOsGZmZgk4wZqZmSXgBGtm\n",
       "ZpaAE6yZmVkCTrBmZmYJOMGamZkl4ARrZmaWgBOsmZlZAskTrKSFku6WdI+kk6e4/lZJd0i6U9IN\n",
       "kvZJHZOZmVlqSROspDnAOcBCYG/gGEl71RS7D3hNROwDfBj4l5QxmZmZtUPqGuwC4N6IWBERa4Fl\n",
       "wBHFAhFxU0T8IT+8BdgxcUxmZmbJpU6wOwArC8cP5OfqeQdwVdKIzMzM2mDTxPePsgUlHQicALxq\n",
       "ygIXXYQOPHAkPxqNiNHZBmdm1k8kDQPDHQ7DcooonQObv7n0CmAkIhbmxx8C1kfEmTXl9gEuAxZG\n",
       "xL1T3Cf49vXE4a9WsmDNzPqMpIgIf252SOoEuynwa+Ag4EHgVuCYiFheKDMfuA54W0TcXOc+AUEE\n",
       "/otiZlaSE2xnJW0ijoh1khYB1wBzgPMjYrmkE/PrS4HTgQpwriSAtRGxIGVcZmZmqSWtwbZKXoMd\n",
       "j2Co07GYmfUK12A7q2cSrP+SmJk1x5+dneWlEs3MzBJwgjUzM0vACdbMzCwBJ1gzM7MEnGDNzMwS\n",
       "cII1MzNLwAnWzMwsASdYMzOzBJxgzczMEnCCNTMzS8AJ1szMLAEnWDMzswScYM3MzBJwgjUzM0vA\n",
       "CdbMzCwBJ1gzM7MEnGDNzMwScII1MzNLwAnWzMwsASdYMzOzBJxgzczMEnCCNTMzS8AJ1szMLAEn\n",
       "WDMzswScYM3MzBJwgjUzM0vACdbMzCwBJ1gzM7MEnGDNzMwScII1MzNLwAnWzMwsASdYMzOzBJxg\n",
       "zczMEnCCNTMzSyBpgpW0UNLdku6RdHKdMp/Nr98hab+U8ZiZmbVLsgQraQ5wDrAQ2Bs4RtJeNWUO\n",
       "BXaPiD2AdwLnpoqnX0ga7nQM3cLPYiM/i438LKxbpKzBLgDujYgVEbEWWAYcUVPmcOBLABFxC7Ct\n",
       "pOcljKkfDHc6gC4y3OkAushwpwPoIsOdDsAM0ibYHYCVheMH8nPTldkxYUxmZmZtkTLBRslymuHr\n",
       "zMzMutamCe+9CtipcLwTWQ21UZkd83OTSHLizUla3OkYuoWfxUZ+Fhv5WVg3SJlgbwP2kLQL8CBw\n",
       "NHBMTZnLgUXAMkmvANZExCO1N4qI2lqumZlZV0uWYCNinaRFwDXAHOD8iFgu6cT8+tKIuErSoZLu\n",
       "Bf4IvD1VPGZmZu2kCLe8mpmZtVpXreTkhSk2mu5ZSHpr/gzulHSDpH06EWc7lPl7kZd7maR1kt7c\n",
       "zvjapeS/j2FJt0v6haTRNofYNiX+fcyTdLWkn+fP4vgOhNkWki6Q9IikuxqUGYjPza4TEV3xRdaM\n",
       "fC+wC7AZ8HNgr5oyhwJX5d+/HLi503F38Fn8FfCf8u8XDvKzKJS7DrgS+NtOx92hvxPbAr8EdsyP\n",
       "53U67g4+ixHg49XnAKwGNu107ImexwHAfsBdda4PxOdmN351Uw3WC1NsNO2ziIibIuIP+eEt9O/8\n",
       "4TJ/LwDeC/wr8Pt2BtdGZZ7DscA3IuIBgIh4rM0xtkuZZ/EQsE3+/TbA6ohY18YY2yYirgfGGxQZ\n",
       "lM/NrtNNCdYLU2xU5lkUvQO4KmlEnTPts5C0A9kHbHWpzX4cWFDm78QewJCkH0i6TdJxbYuuvco8\n",
       "i/OAF0t6ELgDeF+bYutGg/K52XVSTtNplhem2Kj0zyTpQOAE4FXpwumoMs/ibOCUiAhJYvLfkX5Q\n",
       "5jlsBuwPHARsCdwk6eaIuCdpZO1X5lmcCvw8IoYl7QZ8V9K+EfF44ti61SB8bnadbkqwLV2YoseV\n",
       "eRbkA5vOAxZGRKMmol5W5lm8lGwuNWT9bYdIWhsRl7cnxLYo8xxWAo9FxJPAk5J+BOwL9FuCLfMs\n",
       "Xgl8FCAifiPpfmBPsvn5g2ZQPje7Tjc1EW9YmELS5mQLU9R+QF4O/D1Ao4Up+sC0z0LSfOAy4G0R\n",
       "cW8HYmyXaZ9FRLwgInaNiF3J+mHf1WfJFcr9+/g28GpJcyRtSTag5VdtjrMdyjyLu4GDAfL+xj2B\n",
       "+9oaZfcYlM/NrtM1NdjwwhQblHkWwOlABTg3r7mtjYgFnYo5lZLPou+V/Pdxt6SrgTuB9cB5EdF3\n",
       "Cbbk34mPARdKuoOsIvHBiBjrWNAJSboE+GtgnqSVwGKy7oKB+tzsRl5owszMLIFuaiI2MzPrG06w\n",
       "ZmZmCTjBmpmZJeAEa2ZmloATrJmZWQJOsGZmZgk4wVpPk/TnfHu26tf8BmWfaMH7XSTpvvy9fppP\n",
       "3G/2HudJelH+/ak1126YbYxm1h08D9Z6mqTHI2LrVpdtcI8LgSsi4jJJfwOcFRH7zuJ+s47JzLqT\n",
       "a7DWVyQ9W9L38trlnZIOn6LM8yX9KK+F3iXp1fn510m6MX/tpZKeXe9t8v9fD+yev/b9+b3ukvS+\n",
       "Qiz/P9/0+y5JR+bnRyW9VNIZwBZ5HBfn157I/79M0qGFmC+S9GZJm0j6hKRb882z39maJ2dmrdY1\n",
       "SyWazdBrSKNBAAACY0lEQVQWkm7Pv78POAp4U0Q8LmkecBOT16k9Frg6Ij4maRNgy7zsPwMHRcST\n",
       "kk4G3g98uMF7/1fgTkn7A8eT7VO6CXCLpB8CuwGrIuINAJKq+5MGEBFxiqT3RMR+hXtWm5SW5T/L\n",
       "Vfl6u68FTgT+gWwt2QWSngX8WNK1EbGi5PMyszZxgrVe92QxQUnaDPi4pAPI1uPdXtJzI+LRwmtu\n",
       "BS7Iy34rIu6QNAzsDdyYr+28OXDjFO8n4BOSTgMeJduL92+Ay/JdbJB0GXAAcDVwVl5TvTIiftzE\n",
       "z3U18Jk8uR4C/DAinpb0OuAvJb0lL7cNWS16RRP3NrM2cIK1fvNWsi3r9o+IP+fblM0tFoiI6/ME\n",
       "fBhwkaRPAePAdyPi2GnuH8A/RcRl1ROSDmbifpvK3ibukbQf8AbgI5K+HxGNasTFGJ+SNAq8nqwm\n",
       "e0nh8qKI+G6Z+5hZ57gP1vrNNsCjeXI9ENi5tkA+0vj3EfFF4IvAfsDNwKuUbc5d7T/do8571G5e\n",
       "fT3wRklb5P22bwSul/R84KmI+ApwVv4+tdZKqveL7teAE9hYG4ZsB5l3V18j6YX51nRm1mVcg7Ve\n",
       "VzsM/ivAFZLuJNs3dPkUZQ8E/knSWuBx4O8j4jFJxwOX5H2bkPXJTrVZ+YT3jIjbJV1E1vQM2TZx\n",
       "d+TNuZ+QtB5YC/yPKe71L2T9uD+NiONq7n0tcDFZM/a6/NwXgV2Anylry34UeNMU9zWzDvM0HTMz\n",
       "swTcRGxmZpaAE6yZmVkCTrBmZmYJOMGamZkl4ARrZmaWgBOsmZlZAk6wZmZmCTjBmpmZJfAfRKOO\n",
       "nk5L1vYAAAAASUVORK5CYII=\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1130f7e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_curve('', Y_test, logreg_counts_l2_predicted, label='freq')\n",
    "roc_curve('', Y_test, binreg_bincounts_l2_predicted, label='binary')\n",
    "roc_curve('', Y_test, logreg_logcounts_l2_predicted, label='logfreq')\n",
    "roc_curve('', Y_test, logreg_counts_l2_predicted_tfidf, label='tf-idf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_vec = TfidfVectorizer(norm=None)\n",
    "matrix_tfidf = tfidf_vec.fit_transform(all_texts).toarray()\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_tfidf, target)\n",
    "logreg_counts_l2_predicted_tfidf, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression ( freq ) AUC =  0.933953033268\n",
      "Logistic Regression ( binary ) AUC =  0.945350026686\n",
      "Logistic Regression ( logfreq ) AUC =  0.943871197296\n",
      "Logistic Regression ( tf-idf ) AUC =  0.936988525173\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAdgAAAEPCAYAAADh4tUOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXHWZ5/HPlyQYkGRTTdThFkBABHdAUCMiSEcQAyJR\n",
       "R0BQHMRZGRXH1+IoiCzpjIqwoqLiIosgDioBRzSBYQIoXcgdEQQcggNCNISbpDsmaIDEPPPHOdWp\n",
       "rlRVn+ruU7f+vl+vJnXq/OrU04fuevp3V0RgZmZm42uzVgdgZmbWjZxgzczMcuAEa2ZmlgMnWDMz\n",
       "sxw4wZqZmeXACdbMzCwHuSZYSZdIelrSA3XKfEPSw5Luk7RPnvGYmZk1S9412O8Cc2udlHQ4sGtE\n",
       "7AZ8BLgg53jMzMyaItcEGxE3A4N1ihwJfC8teycwQ9Ir8ozJzMysGVrdB7sdsLzs+HFg+xbFYmZm\n",
       "Nm5anWABVHHstRvNzKzjTW7x+68Adig73j59bhhJTrpmZqMQEZWVGGuSVifYxcDJwEJJ+wGrIuLp\n",
       "agX9Q5KQ1BcRfa2Oox20873QAg0AhREL7r8Ipkwf+xtefPHzcdllW4z9Qq0lke2+1dWXfuViMIKe\n",
       "vC4+3lw5aa1cE6yky4GDgJmSlgPzgSkAEXFhRFwr6XBJjwB/Bj6UZzxmY5U5ccJgzA8Vrr56w6pp\n",
       "02r+cVhYvZqBQ+c0GsYgET1QnpCKU/X9ruheGYzYpNuoIdKCvoi+vnGKx2zUck2wEXFshjIn5xmD\n",
       "TVwNJMNGDMb8itYUJe/Ts2gRg9OHaqMFFYsxAyJ6e+snjCpbRo5QkyugoWQ6GIGcVMzaT6ubiK1x\n",
       "xVYH0EaK1Z4sS6ybJEMVi2NNugUVi8MzYn9/6dFg9PaOufkwTa40WJMrjvV9u0ix1QGYAagTNlyX\n",
       "FO6D7U7jkPAaUli9moF588brcoNE9IxPv+Hw63ZSP5+1L392tpZrsNYyKhYHWLdmBrcd2dDrBs4m\n",
       "Cs8P1e6G+iMzG+MflWUJtdRUO+Z+Q7Nm8uCn8VXrjxgnWGulArcdySZ9mjDUr1njdasaTqrjZJTN\n",
       "t2ZtxzXb8VHvjxUnWGuJtGl4kMokmibWQWZET+1VNssH+TSbm2/NLBP3wdq4qxy9u/Js6Hm+okx/\n",
       "PzGn6vSUUr9muJZoNja1Pjv9mTp+6t1LJ1gbdwNbKHa9YtiUlU2tngzzDqh3GdcUzcbICTZ/TrCW\n",
       "jyr9pGVzQQeZ0wul6TJOlmZN14kJVtLuwBXAK4HTI+L8FodUV7176T5Ya0xZUi0sXhyrpk0bfv7F\n",
       "NbDkPXDOQCmxtuUvsZm1rc8AP4+I17Y6kLFyDdZqGmmZP1ZPhtuOhC3KBiOtnRFx9mA77NJkNuF1\n",
       "aA32Z8DlEXFxlXObRcSGFoRVk5uIrb4aU2LU309pmb+yuZ/Dmnu1QFF1mo2ZtVynJVhJNwJvAdYB\n",
       "60k2hFkN7Jg+fyTwEPBN4EDgOeBrEfHN9PVbABek5Z4ELgX+KSJ2ICduIra6ehYtKtQYkFQ+T6bg\n",
       "5l4zy1NEvFVSP3BZRFwi6VLgWOCwiLg9TaC3AD8BjiHZ7vRnkn4bEdeTbCizM0n/7VbAElq4x7gT\n",
       "7ARUuTzhDCmit3dYs67EAKf2FHTTYPLD2QdaUPUHteZkVTPrTBqneebj9Ef5TyPi9vTxXsDMiPhC\n",
       "evyYpO8A7wOuB44CPhoRq4BVkr4OnDkOMYyKE+wEowUa4KD+AjfNIfpAfbAKpHsqfqH6gGSxfI/+\n",
       "NZtg2qi1KoAVZcc7AttKKv/DfhLwi/TxtsDysnN/yDe8+pxgO9xQ3+ipPUODjWbsu4hV02rMQT2o\n",
       "P1nwvg9wAjWz9lf+x/8fgMci4lU1yj4JzAKWpsez8gxsJE6wHUqnFTawxSqlNU3Yf3EwJRnxq9Wr\n",
       "Gc0epGZmbabyc+wuYI2kz5AMdHoR2AOYGhF3A1cCn5V0J0kf7CdwH6xlMZRUAZgxbPSuisUoW3pw\n",
       "0AnUzLpAUJYgI2KDpCOArwCPAi8hGVV8RlpkAfBt4DGSpuVLgU82Md5hPE2njQ3bZ/TUHnjr92Dz\n",
       "aVXLzlizJgbf+U7PPzWzIZ02TWe8SeolGZHsaTqWkBhYSU8hygbo9uy/CJ4PBt6+yQL5je+HamZm\n",
       "uXOCbU+FHgYhQkNTatatJt52pNz0a2bWkJZ9aLpJsY0MqrABKQIxMDVZJQkocNMcuG2e55uamTUg\n",
       "IooR0bKRxK7Btlj53qlBMi+V0vSZpPaKp9KYmXUeD3JqogwbkQ/1p6qUXHt7nVzNbFQm+iCnZvAg\n",
       "p/ZRGJpakyywDxE96WhhKvZMLYw4l9XMzNqWE+x4qbEjTbkA6FOpyaB89O+whfTT2qv7XM3MOpib\n",
       "iMeLFOpjkOFJdsSlCEu1V/qLlL120E3DZjZWbiLOn5uIx8PINdRBypuAs1p0S4Hp6yFJqv6BN7MJ\n",
       "S9Iy4MMR8fOK5w8ELoqIV7cksFFygh3JxsQ6SNlfKZUDlobKbPJyqifmRbfA9PWgyZtsFWdmNkEN\n",
       "Wxpx6MmIm4GOSq7gBJtFgerV/6y11aoblau4PlxjNTNrHUkCiJz6Sp1g60lqr6MebKRicYB+UNEb\n",
       "lZuZZTRb0jeBbYCfAh8F3kTZmsJpU/I3gQ+S7BG7BPj7iHhB0gzg+8Bskhx3K/CPEbEifW0RuAWY\n",
       "A+wDnCnpfRHx+lIAkk4B3hIR7xrLN+IEW0s6jUZ9wAI1nCAlBuinwJzewYrpN2ZmVp2A44BDgb8A\n",
       "V5PslPOzinIBHAW8HXiBJImeAFxIskLhxcB7SXLcJcD5wLvLXv8B4DDgt2mZz0p6dUQ8lJ4/HviX\n",
       "sX4zTrC1JU3DCxTVmoIlBtRXZ43LxbcE4ORqZh1H1SsVDWt40GeSOM8vq21+kaSmWplgAb4REU+l\n",
       "5a4GXgsQEQPAT0qFJJ0F3FjxHpdGRGlT9hclXUmSdM+Q9BqSWvE1Dca+CSfYcsNHCg+mA5kGN54e\n",
       "NmBpsLJvdWhh/sQqT7Uxs040isQ4npaXPf4DsG2Nck+VPV5bKidpS+BrJLXb0ufxVkrn01R5D4Dv\n",
       "AT8kqS0fD1wREetG/R2knGBLNq6sJBgaJTy0DnDZaktVBiwNJVZPtTEzG5tZFY+faPD1nwJeBcyO\n",
       "iGckvRa4h6T5uZRgh9XQI+IOSS9KegtwbPo1Zk6wG1WOFh4aJVxjKUMnVjOz8SXg45KuIamVfg5Y\n",
       "2OA1tkpf+ydJPcD8Gu9T6TKSvtoXI+K2Bt+zKifYMhX9DoNQO7mmvF6wmdn4CeAHwPUkTb4/Bb4A\n",
       "7Ef9fV3L58+eR9Lc+yywAvgqcGSV8pUuIxnYNObBTSW5LpUoaS7JNzsJ+E5EnFNxfibJcOq/IUn2\n",
       "50bEpVWuk8uyXqXFIlaeDYIorI1NFnyQiGrNwgAqFj2X1czalpdKzE7SFsDTwD4R8bsGXlfzXua2\n",
       "gpCkSSTV7bnAnsCxkvaoKHYycG9EvBboBb4iqZm16kLMD/U8D6XkKjEgEaUvPF/VzGwi+ChwVyPJ\n",
       "dSR5JrPZwCMRsQxA0kJgHrC0rMyTwF7p4+nAyohYn2NMWdRYealYbclDJ18zsw6XLlwRwJgWlqiU\n",
       "Z4LdjuFDoR8H3lhR5iLgRklPANOAo3OMZ5jKKTgZuL/VzKwLRcROeVw3zwSbpXP3dODXEdEraRfg\n",
       "Bkl7R8SayoKS+soOixFRHGN8je98Y2bWxiT1knS3WRvIM8GuAHYoO96BpBZbbn/giwAR8TtJjwG7\n",
       "A3dXXiwi+vIJM1G2iMTGhSWGNwu7OdjM2lpa8SiWjiVVm6JiTZJngr0b2E3STiQThY9h08m7DwGH\n",
       "ALdKegVJcn00x5g2ITEwNPN4075XNwubmdmo5DaKOB2sdDJwHfAgydJTSyWdJOmktNhZwOsl3Uey\n",
       "1uRn0nUkc7fybECKQMlCEdUXkXCt1czMRiXXebDjZdzmbJWtNTwwFbZ+PohAtUYIey1hM+tkngeb\n",
       "v3r3cuKs5LRx+7lBoMDaGcE5rErPuinYzKzF0ukyH46In4/hGlsAVwIHAtdFxDHjFF7DJk6Crdh+\n",
       "Ll2hyTVUM7P2Ub7k4Wi9F3g50BMRG8Ye0ujl1gfbjtK1ht2vambWvXYE/qtWck1XGWyKCZVgY36I\n",
       "vqB8CUQPZjIzay+SNpd0nqQV6dfXJG1edv4zkp6Q9Likf5C0QdIukhYA/wc4RtIaSSdKOkHSrZK+\n",
       "KulZYH56/XMl/V7SU5IukDS17PqfLrv+ien1X9no9zGhEmwqWQqxv4iKxQDwYCYzs7Yhko3PZwN7\n",
       "p1+z0+dKm8j8b+BgYDc2LqwRETGfZHbKwoiYFhGXpOdmA78jaTo+CzgH2DW99q4kKw+eWXb9T5FM\n",
       "IX1V+u+omq0nYoItKURvr5xczcwqSDEuX6N3HPAvEfFsRDwLLACOT88dDVwSEUsjYi2b7vcqNt3v\n",
       "9YmI+FbabPwC8L+AUyJiVUQ8B3wJeF/F9R+MiL+k1x/VINjuH+SUTs0ZmCq2XnxL0L8eFb1LjplZ\n",
       "Ta2fwrMt8Puy4z+kzwFsA9xVdq5yhcBqytfFfxmwJfAraejbFBsrnNsAv6x471Hp/gQLBfUBfRtg\n",
       "ehFPxzEza3tPADuxcfe1WSTL70KyC1vlMrwjKa9NPwusBfaMiCerlH0yfb+SWVXKZDKRm4jNzKw9\n",
       "XQ6cIWmmpJkk/aPfT89dCXxI0qslbUkyqCmztJn4IuA8SS8DkLSdpEPLrn+CpD3S6496PefuTrDS\n",
       "wODUtDl40S3gZmEzs3YXwBdI1rO/P/26O32OiFgCfAPoB/4LuD193Qtlr4+K61X2B58KPALcIelP\n",
       "wA0kA5pK1z8PuDG9/ugXvejqpRKlUB+DMT96VCyGm4fNbCKZCEslStoDeADYPK+FJSRtAHaNiE02\n",
       "o6l3L7u2BptuqA4H9SfTcdZMbv+/JMzMbESS3i3pJZIKJFNuFrd61aZqunaQ08qzh/Z2LTCnl4ju\n",
       "/WPCzGyC+QjwXeCvJPvffizn9xtVBa37Emxpx5ypoP5+KCVZMzPrChFxWJPfb1TLK3Zfgk0X9d/6\n",
       "hsVDqzRp7ItHm5mZNaS7mk3TUcNaoGDKdC+BaGZmLdM9CTbd77XnNBTzu2N0nJmZda7uSLBpciWi\n",
       "Z/jTDJTvnGNmZtYs3dIHW6ixdmayc46ZmVmTdUcNNqUFGmD/xZFuQ+daq5lZh0vnvC5P93fdu+Lc\n",
       "rPT5qhUpSX2SLstyrTx0Sw22pMCUaUML+nv0sJlZZ5G0DDgxIm5MnzoX+FhEXF1ZNiL+AEyrc7nK\n",
       "HFDzWnnoqhos+y+CtOYqMYBrsWZmnSZI919Na6azgAfHetHxvFZW3ZVgh0/NKUTgaTpmZh0ibc6d\n",
       "BVwt6S/Ac8Ak4D5JD1cpv5OkDZI2S493lnSTpNWSrgdmps9vDqypd608dFeCNTOzjhURx5NscH5E\n",
       "RGwZES9NT+0VEbtluMQPSTZL3xr4PPD3yWXjxYjYqsFrjVnX9MFqgQY4qL/UNFxah9jMzBqUDhQd\n",
       "s2buYCZpFvB64K0RsQ64WdLV0LqZJB2fYLVAAxU/CZ6aY2Y2Bu26taek59i4v+ueFae3BQYjYm3Z\n",
       "c78HdmhSeJvohibiAkDMD/e3mpl1vpq154jYKiKmRcT0iHi84vSTQEHSlmXP7ZhLhBl1Q4IFhkYN\n",
       "g5uGzcw62dPALo2+KCJ+D9wNLJA0RdIBwBG0cLpm1yRYSjVZjxw2M+tkXwLOkDQo6VOMnCDLzx8H\n",
       "vBEYAM4EvlenbO46vg+2jPd9NTPrcBGxGFhc9tRX6pRdRjL1pnT8GPCWOuVHta/raHVNDdY1VzMz\n",
       "aycdnWC1QEOrNalY9MpNZmbWNjIlWElbSto972BGoVA2erjgDdbNzKxdjJhgJR0J3Atclx7vI2lx\n",
       "/Vc1kTQw4K5XMzNrM1lqsH0ko7IGASLiXuCVOcbUqMLWQzN0zMzM2kOWBLsuIlZVPLchy8UlzZX0\n",
       "kKSHJZ1ao0yvpHsl/UZSMct1q3Dfq5mZtZUs03T+U9L7gcmSdgP+CbhtpBdJmgScDxwCrAB+KWlx\n",
       "RCwtKzMD+Bbw9oh4XNLMrIFrgQZYOyNglSLoUdF7v5qZZSXJn5k5y5JgPwF8DngBuJykL/bzGV43\n",
       "G3gknaeEpIXAPGBpWZnjgB+XlryKiGczRw6FlecIXHs1M2tIRLTlWsPdJkuC3T0iTgdOb/Da2wHL\n",
       "y44fJ+nLLbcbMEVSP8mu9F+PiMvqXTSdmlMABnsYLBDR4yk6ZmbWbrIk2K9K+hvgR8AVEfGbjNfO\n",
       "0vwwBdgXOBjYErhd0h0RUW8z3ELMT//66htq4ii06+4PZmY2MY2YYCOiV9I2wNHAhZKmA1dGxEjN\n",
       "xCsYvk3QDiS12HLLgWfT7YXWSvoFsDdQbef6Pg7kVOB59ak34KoBCl6+ycwsJakX6G1xGJZSRPZ+\n",
       "bkl/C5wKHBMRU0YoOxn4LUnt9AngLuDYikFOryYZCPV24CXAnem1H6y4VkSEtEAxVHuVQgQRSMVi\n",
       "uAZrZjZc6bOz1XFMVCPWYCXtSVJ7fS+wErgCOGWk10XEekknkwyKmgRcHBFLJZ2Unr8wIh6StAS4\n",
       "n2Tqz0WVydXMzKwTZemDvQRYSDKVZkUjF4+I/wD+o+K5CyuOzwXObeS6ZmZm7S5LH+x+zQjEzMys\n",
       "m9RMsJJ+FBFHSXqgyumIiL1yjMvMzKyj1avBfjL99wigspO8bVYA8RxYMzNrRzXXIo6IJ9KHH4uI\n",
       "ZeVfwMeaEl023qbOzMzaTpbF/g+t8tzh4x2ImZlZN6nXB/tRkprqLhX9sNOAW/MOzMzMrJPV64P9\n",
       "IckUm7NJFpco9cOuiYiVeQdmZmbWyeol2IiIZZI+TsWgJkk9EeFdzs3MzGqol2AvB94B/Irqo4Z3\n",
       "ziUiMzOzLlAzwUbEO9J/d2paNGZmZl1ixFHEkt4saav08fGSvippx/xDMzMz61xZpul8G/iLpL1J\n",
       "Fvl/FPjXXKPKatEt4EUmzMysDWVJsOsjYgPwLuBbEXE+yVSd1pu+Hi8yYWZm7SjLbjprJJ0OfAA4\n",
       "UNIkoO5esGZmZhNdlhrsMcALwIkR8RSwHfDlXKMyMzPrcCMm2Ih4EvgBMEPSEcDzEdHyPtieRYtg\n",
       "dZYKuJmZWfNlGUV8NHAncBRwNHCXpKPyDmwkg9Onw7wDWh2GmZlZVVmqgGcAb4iIZwAkvQz4OfCj\n",
       "PAMzMzPrZFn6YAX8sex4JZvuD2tmZmZlstRglwDXSfohSWI9hmQTADMzM6thxAQbEZ+W9B6g1OF5\n",
       "YUT8JN+wzMzMOlu9/WBfRTIdZ1fgfuDTEfF4swIzMzPrZPX6YC8BrgH+DrgH+EZTIjIzM+sC9ZqI\n",
       "t4qIi9LHD0m6txkBmZmZdYN6CXaqpH3TxwK2SI9Fshn7PblHNzIv9G9mZm2pXoJ9CvhKneM5uUQ0\n",
       "EmkAKNDfTwRe6N/MzNpSvQ3Xe5sYRyMKRIhiMVodiJmZWS1ZFppoOxJOrmZm1tY6JsFqgQZI+1wj\n",
       "vJKUmZm1t45JsEAh5of7XM3MrCNk2U1nM0nHSzozPZ4laXb+oZmZmXWuLDXY/we8CTguPX4ufa7p\n",
       "JAZa8b5mZmaNyrLY/xsjYp/SQhMRMSBpSs5xVbWSngKe+2pmZh0gSw32RUmTSgfpfrAb8gupth4G\n",
       "IdwPa2Zm7S9LDfabwE+Al0s6C3gvySbsLaFiMVlowjVZMzNrY1m2q/u+pF8BB6dPzYuIpfmGVVch\n",
       "ens9TcfMzNpallHEs4A/A1enX39OnxuRpLmSHpL0sKRT65R7g6T16b6zVa08GwYoZHlbMzOzlsvS\n",
       "RHwtDK2cNBXYGfgt8Jp6L0r7bc8HDgFWAL+UtLiy9puWOwdYArUXkOh5HpJBxMUMIZuZmbVWlibi\n",
       "/1l+nO6o8/EM154NPBIRy9LXLQTmAZXNy58A/g14Q4ZrmpmZdYSGV3JKt6l7Y4ai2wHLy44fT58b\n",
       "Imk7kqR7QenyjcZjZmbWjkaswUr6VNnhZsC+JE2+I8mSLM8DTouIkCTqNBGbmZl1kix9sFuVPV4P\n",
       "XAP8OMPrVgA7lB3vQFKLLfc6YGGSW5kJHCZpXUQsrrxYX+m/ly5Dc+b0RkQxQwxmZhOGpF6gt8Vh\n",
       "WEoRtSua6QCk/xsRn6pZqPZrJ5MMhjoYeAK4Czi21hQfSd8Fro6Iq6qciwBEQH8RT9MxMxuZpIgI\n",
       "f162SM0arKTJEbFe0puV/l9q5MLpa08GrgMmARdHxFJJJ6XnLxxT5GZmZm2sZg1W0j0Rsa+kbwPb\n",
       "Aj8C/pKejmo1zby4Bmtm1jjXYFurXh9s6X/KVGAl8NaK801LsGZmZp2mXoJ9maRTgAeaFYyZmVm3\n",
       "qJdgJwHTmhWImZlZN6mXYJ+KiAVNi8TMzKyLNLySU6sMTG11BGZmZtnVS7CHNC2KDLY+rdURmJmZ\n",
       "ZVczwUbEymYGYmZm1k06ponYzMyskzjBmpmZ5aCzEuyiWwAGWx2GmZnZSDonwe6/CIDo7e1pcSRm\n",
       "ZmYjyrJdXXuYMh0OPWDQW7KbmVkn6JwaLBCBa69mZtYROirBmpmZdQonWDMzsxx0ToJdt7rVEZiZ\n",
       "mWXWOQn2tnmtjsDMzCyzzkmwZmZmHcQJ1szMLAdOsGZmZjlwgjUzM8uBE6yZmVkOnGDNzMxy4ARr\n",
       "ZmaWAydYMzOzHHROgl1baHUEZmZmmSmi/fd/kxQQRKBWx2Jm1ikkRUT4c7NFOqcGa2Zm1kGcYM3M\n",
       "zHLgBGtmZpYDJ1gzM7McOMGamZnlwAnWzMwsB06wZmZmOXCCNTMzy4ETrJmZWQ5yT7CS5kp6SNLD\n",
       "kk6tcv79ku6TdL+kWyXtlXdMZmZmect1qURJk4DfAocAK4BfAsdGxNKyMm8CHoyIP0maC/RFxH4V\n",
       "1/FSiWZmDfJSia2Vdw12NvBIRCyLiHXAQmBeeYGIuD0i/pQe3glsn3NMZmZmucs7wW4HLC87fjx9\n",
       "rpYPA9fmGpGZmVkTTM75+pnbnyXNAU4E3ly9RB/Sgr70oBgRxTHGZmbWVST1Ar0tDsNSeffB7kfS\n",
       "pzo3Pf4ssCEizqkotxdwFTA3Ih6pch33wZqZNch9sK2VdxPx3cBuknaStDlwDLC4vICkWSTJ9QPV\n",
       "kquZmVknyrWJOCLWSzoZuA6YBFwcEUslnZSevxA4EygAF0gCWBcRs/OMy8zMLG+5NhGPl7SJeDCC\n",
       "nlbHYmbWKdxE3Fodk2D9Q2Jm1hh/draWl0o0MzPLgROsmZlZDpxgzczMcuAEa2ZmlgMnWDMzsxw4\n",
       "wZqZmeXACdbMzCwHTrBmZmY5cII1MzPLgROsmZlZDpxgzczMcuAEa2ZmlgMnWDMzsxw4wZqZmeXA\n",
       "CdbMzCwHTrBmZmY5cII1MzPLgROsmZlZDpxgzczMcuAEa2ZmlgMnWDMzsxw4wZqZmeXACdbMzCwH\n",
       "TrBmZmY5cII1MzPLgROsmZlZDpxgzczMcuAEa2ZmlgMnWDMzsxw4wZqZmeXACdbMzCwHTrBmZmY5\n",
       "cII1MzPLgROsmZlZDnJNsJLmSnpI0sOSTq1R5hvp+fsk7ZNnPGZmZs2SW4KVNAk4H5gL7AkcK2mP\n",
       "ijKHA7tGxG7AR4AL8oqnW0jqbXUM7cL3YiPfi418L6xd5FmDnQ08EhHLImIdsBCYV1HmSOB7ABFx\n",
       "JzBD0ityjKkb9LY6gDbS2+oA2khvqwNoI72tDsAM8k2w2wHLy44fT58bqcz2OcZkZmbWFHkm2MhY\n",
       "TqN8nZmZWduanOO1VwA7lB3vQFJDrVdm+/S5TUhy4k1Jmt/qGNqF78VGvhcb+V5YO8gzwd4N7CZp\n",
       "J+AJ4Bjg2Ioyi4GTgYWS9gNWRcTTlReKiMparpmZWVvLLcFGxHpJJwPXAZOAiyNiqaST0vMXRsS1\n",
       "kg6X9AjwZ+BDecVjZmbWTIpwy6uZmdl4a6uVnLwwxUYj3QtJ70/vwf2SbpW0VyvibIYsPxdpuTdI\n",
       "Wi/pPc2Mr1ky/n70SrpX0m8kFZscYtNk+P2YKWmJpF+n9+KEFoTZFJIukfS0pAfqlJkQn5ttJyLa\n",
       "4oukGfkRYCdgCvBrYI+KMocD16aP3wjc0eq4W3gv3gT8j/Tx3Il8L8rK3QhcA/xdq+Nu0c/EDOA/\n",
       "ge3T45mtjruF96IP+FLpPgArgcmtjj2n+3EgsA/wQI3zE+Jzsx2/2qkG64UpNhrxXkTE7RHxp/Tw\n",
       "Trp3/nCWnwuATwD/BvyxmcE1UZb7cBzw44h4HCAinm1yjM2S5V48CUxPH08HVkbE+ibG2DQRcTMw\n",
       "WKfIRPncbDvtlGC9MMVGWe5FuQ8D1+YaUeuMeC8kbUfyAVtaarMbBxZk+ZnYDeiR1C/pbknHNy26\n",
       "5spyLy4CXiPpCeA+4JNNiq0dTZTPzbaT5zSdRnlhio0yf0+S5gAnAm/OL5yWynIvzgNOi4iQJDb9\n",
       "GekGWe7DFGBf4GBgS+B2SXdExMO5RtZ8We7F6cCvI6JX0i7ADZL2jog1OcfWribC52bbaacEO64L\n",
       "U3S4LPeCdGDTRcDciKjXRNTJstyL15HMpYakv+0wSesiYnFzQmyKLPdhOfBsRKwF1kr6BbA30G0J\n",
       "Nsu92B/4IkBE/E7SY8DuJPPzJ5qJ8rnZdtqpiXhoYQpJm5MsTFH5AbkY+CBAvYUpusCI90LSLOAq\n",
       "4AMR8UgLYmyWEe9FRLwyInaOiJ1J+mE/2mXJFbL9fiwCDpA0SdKWJANaHmxynM2Q5V48BBwCkPY3\n",
       "7g482tQo28dE+dxsO21Tgw0vTDEky70AzgQKwAVpzW1dRMxuVcx5yXgvul7G34+HJC0B7gc2ABdF\n",
       "RNcl2Iw/E2cB35V0H0lF4jMRMdCyoHMk6XLgIGCmpOXAfJLuggn1udmOvNCEmZlZDtqpidjMzKxr\n",
       "OMGamZnlwAnWzMwsB06wZmZmOXCCNTMzy4ETrJmZWQ6cYK2jSfpruj1b6WtWnbLPjcP7XSrp0fS9\n",
       "fpVO3G/0GhdJenX6+PSKc7eONUYzaw+eB2sdTdKaiJg23mXrXOO7wNURcZWktwHnRsTeY7jemGMy\n",
       "s/bkGqx1FUkvlfSztHZ5v6Qjq5TZRtIv0lroA5IOSJ8/VNJt6WuvlPTSWm+T/nszsGv62lPSaz0g\n",
       "6ZNlsfwwcALWAAACnUlEQVR7uun3A5KOSp8vSnqdpLOBLdI4LkvPPZf+u1DS4WUxXyrpPZI2k/Rl\n",
       "SXelm2d/ZHzunJmNt7ZZKtFslLaQdG/6+FHgaODdEbFG0kzgdjZdp/Y4YElEnCVpM2DLtOzngIMj\n",
       "Yq2kU4FTgM/Xee93AvdL2hc4gWSf0s2AOyXdBOwCrIiIdwBIKu1PGkBExGmSPh4R+5Rds9SktDD9\n",
       "Xq5N19t9K3AS8A8ka8nOlvQS4BZJ10fEsoz3y8yaxAnWOt3a8gQlaQrwJUkHkqzHu62kl0fEM2Wv\n",
       "uQu4JC3704i4T1IvsCdwW7q28+bAbVXeT8CXJZ0BPEOyF+/bgKvSXWyQdBVwILAEODetqV4TEbc0\n",
       "8H0tAb6eJtfDgJsi4gVJhwJ/K+m9abnpJLXoZQ1c28yawAnWus37Sbas2zci/ppuUza1vEBE3Jwm\n",
       "4COASyV9FRgEboiI40a4fgD/HBFXlZ6QdAjD99tU8jbxsKR9gHcAX5D084ioVyMuj/F5SUXg7SQ1\n",
       "2cvLTp8cETdkuY6ZtY77YK3bTAeeSZPrHGDHygLpSOM/RsR3gO8A+wB3AG9Wsjl3qf90txrvUbl5\n",
       "9c3AuyRtkfbbvgu4WdI2wPMR8QPg3PR9Kq2TVOsP3SuAE9lYG4ZkB5mPlV4j6VXp1nRm1mZcg7VO\n",
       "VzkM/gfA1ZLuJ9k3dGmVsnOAf5a0DlgDfDAinpV0AnB52rcJSZ9stc3Kh71nRNwr6VKSpmdItom7\n",
       "L23O/bKkDcA64B+rXOv/k/Tj/ioijq+49vXAZSTN2OvT574D7ATco6Qt+xng3VWua2Yt5mk6ZmZm\n",
       "OXATsZmZWQ6cYM3MzHLgBGtmZpYDJ1gzM7McOMGamZnlwAnWzMwsB06wZmZmOXCCNTMzy8F/A7MR\n",
       "ZhD6gYKbAAAAAElFTkSuQmCC\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110ed5790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_curve('', Y_test, logreg_counts_l2_predicted, label='freq')\n",
    "roc_curve('', Y_test, binreg_bincounts_l2_predicted, label='binary')\n",
    "roc_curve('', Y_test, logreg_logcounts_l2_predicted, label='logfreq')\n",
    "roc_curve('', Y_test, logreg_counts_l2_predicted_tfidf, label='tf-idf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import string\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "def stem_tokens(tokens, stemmer):\n",
    "    stemmed = []\n",
    "    for item in tokens:\n",
    "        stemmed.append(stemmer.stem(item))\n",
    "    return stemmed\n",
    "\n",
    "def tokenize(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [i for i in tokens if i not in string.punctuation]\n",
    "    stems = stem_tokens(tokens, stemmer)\n",
    "    return stems\n",
    "\n",
    "\n",
    "count_vect_without_stopwords = CountVectorizer(stop_words='english')\n",
    "matrix_counts_without_stopwords = count_vect_without_stopwords.fit_transform(all_texts).toarray()\n",
    "\n",
    "count_vect_with_lemm = CountVectorizer(tokenizer=tokenize)\n",
    "matrix_counts_with_lemm = count_vect_with_lemm.fit_transform(all_texts).toarray()\n",
    "\n",
    "count_vect_with_lemm_without_stopwords = CountVectorizer(stop_words='english', tokenizer=tokenize)\n",
    "matrix_counts_with_lemm_without_stopwords = count_vect_with_lemm_without_stopwords.fit_transform(all_texts).toarray()\n",
    "\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts, target)\n",
    "logreg_counts_l2_predicted, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_without_stopwords, target)\n",
    "logreg_counts_l2_predicted_without_stopwords, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_with_lemm, target)\n",
    "logreg_counts_l2_predicted_with_lemm, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = cross_valid(matrix_counts_with_lemm_without_stopwords, target)\n",
    "logreg_counts_l2_predicted_with_lemm_without_stopwords, _ = logistic_reg(X_train, X_test, Y_train, C=0.02915728098552524)\n",
    "\n",
    "roc_curve('',Y_test, logreg_counts_l2_predicted, label='all')\n",
    "roc_curve('',Y_test, logreg_counts_l2_predicted_without_stopwords, label='w/o stop-words')\n",
    "roc_curve('',Y_test, logreg_counts_l2_predicted_with_lemm, label='w/ stemming')\n",
    "roc_curve('',Y_test, logreg_counts_l2_predicted_with_lemm_without_stopwords, label='w/ stemming\\nw/o stop-words')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}