3.1 - LASSO - PN and PNU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "pd.options.display.max_rows = 400\n",
    "pd.options.display.max_columns = 400\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MaxAbsScaler\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from epiml.loadepiml import LoadEpiml, load_search, save_search\n",
    "from epiml.epimlsklearn.pnuwrapper import PNUWrapper\n",
    "from epiml.epimlsklearn.nestedcross import NestedCV\n",
    "from epiml.epimlsklearn.jsearchcv import JRandomSearchCV, extract_score_grid\n",
    "from epiml.epimlsklearn.frankenscorer import FrankenScorer, extract_scores_from_nested"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = \"C:\\Data\\membership14_final_0103.txt\"\n",
    "lc = LoadEpiml(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's stratify 80/20 train test holdout for future use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(lc.X, lc.y, test_size=0.2, random_state=771, stratify=lc.y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LASSO pipeline and model on labeled data (PN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimators = [('scaler', MaxAbsScaler()),\n",
    "              ('clf',PNUWrapper(base_estimator=LogisticRegression(penalty='l1', C=2, random_state=732)))]\n",
    "pipe = Pipeline(estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('scaler', MaxAbsScaler(copy=True)), ('clf', PNUWrapper(base_estimator=LogisticRegression(C=2, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l1', random_state=732, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "      num_unlabeled=0.0, pu_learning=False, random_state=None,\n",
       "      threshold_set_pct=None))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.fit(X_train.values, y_train.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scores of LASSO PN Baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'SCORE': 0.89976689976689972,\n",
       "  'assumed_brier': 0.60031267154696111,\n",
       "  'assumed_brier_neg': 0.60285852450719479,\n",
       "  'assumed_f1': 0.0094622917866817001,\n",
       "  'assumed_f1beta10': 0.3170392537956721,\n",
       "  'confusion_matrix_lab': array([[161,  54],\n",
       "         [ 32, 386]]),\n",
       "  'confusion_matrix_un': array([[ 6939, 80783],\n",
       "         [   32,   386]]),\n",
       "  'labeled_acc': 0.86413902053712477,\n",
       "  'labeled_avg_prec': 0.92563531297005985,\n",
       "  'labeled_brier': 0.1040448755848036,\n",
       "  'labeled_brier_neg': 0.17793964147052785,\n",
       "  'labeled_brier_pos': 0.066036802222529167,\n",
       "  'labeled_f1': 0.89976689976689972,\n",
       "  'labeled_prec': 0.87727272727272732,\n",
       "  'labeled_recall': 0.92344497607655507,\n",
       "  'labeled_roc_auc': 0.83614109268944037,\n",
       "  'pr_one_unlabeled': 0.92254333938999167,\n",
       "  'pu_mix_assumed_f1beta10': 32.629912394133711,\n",
       "  'pu_score': 0.92598701456650112},\n",
       " 0.89976689976689972)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FrankenScorer()(pipe, X_test.values, y_test.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Coefficients of features for LASSO PN Baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG4_Sum</th>\n",
       "      <td>31.442457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat54_Sum</th>\n",
       "      <td>7.552092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG69_Sum</th>\n",
       "      <td>5.673218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG5_Sum</th>\n",
       "      <td>5.406727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG10_Sum</th>\n",
       "      <td>4.901706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG19_Sum</th>\n",
       "      <td>4.670333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat61_Sum</th>\n",
       "      <td>3.886519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG31_Sum</th>\n",
       "      <td>3.869930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG71_Sum</th>\n",
       "      <td>3.583528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat58_Sum</th>\n",
       "      <td>3.453970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat98_Sum</th>\n",
       "      <td>2.951269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat87_Sum</th>\n",
       "      <td>2.894050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat30_Sum</th>\n",
       "      <td>2.370636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat43_Sum</th>\n",
       "      <td>2.337440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat13_Sum</th>\n",
       "      <td>1.970817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat59_Sum</th>\n",
       "      <td>1.880306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat6_Sum</th>\n",
       "      <td>1.725292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>1.521835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat45_Sum</th>\n",
       "      <td>1.355579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat76_Sum</th>\n",
       "      <td>1.274270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat15_Sum</th>\n",
       "      <td>1.273675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG4_Sum</th>\n",
       "      <td>1.139761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat66_Sum</th>\n",
       "      <td>1.059264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG46_Sum</th>\n",
       "      <td>0.917191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG86_Sum</th>\n",
       "      <td>0.887100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat42_Sum</th>\n",
       "      <td>0.746189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG14_Sum</th>\n",
       "      <td>0.661754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat67_Sum</th>\n",
       "      <td>0.607018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat40_Sum</th>\n",
       "      <td>0.592419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat1_Sum</th>\n",
       "      <td>0.505068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG62_Sum</th>\n",
       "      <td>0.494281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gender_M</th>\n",
       "      <td>0.486453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat11_Sum</th>\n",
       "      <td>0.472664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG51_Sum</th>\n",
       "      <td>0.355996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat24_Sum</th>\n",
       "      <td>0.211333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG22_Sum</th>\n",
       "      <td>0.205492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat77_Sum</th>\n",
       "      <td>0.132837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat31_Sum</th>\n",
       "      <td>0.090761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REVCODE_FLAG24_Sum</th>\n",
       "      <td>0.085903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat21_Sum</th>\n",
       "      <td>-0.116828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat63_Sum</th>\n",
       "      <td>-0.270736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat37_Sum</th>\n",
       "      <td>-0.470340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG41_Sum</th>\n",
       "      <td>-0.509362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat96_Sum</th>\n",
       "      <td>-0.728530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG11_Sum</th>\n",
       "      <td>-0.867685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG49_Sum</th>\n",
       "      <td>-1.249089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat9_Sum</th>\n",
       "      <td>-1.340142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat34_Sum</th>\n",
       "      <td>-1.846452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG7_Sum</th>\n",
       "      <td>-2.373370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG50_Sum</th>\n",
       "      <td>-2.383799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG10_Sum</th>\n",
       "      <td>-3.181193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat19_Sum</th>\n",
       "      <td>-3.636398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ndc_cat72_Sum</th>\n",
       "      <td>-3.640795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG5_Sum</th>\n",
       "      <td>-4.231993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG55_Sum</th>\n",
       "      <td>-4.962752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG48_Sum</th>\n",
       "      <td>-13.397140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG4_Sum</th>\n",
       "      <td>-13.955133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPT_FLAG6_Sum</th>\n",
       "      <td>-24.699813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIAG_FLAG9_Sum</th>\n",
       "      <td>-28.111479</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0\n",
       "DIAG_FLAG4_Sum      31.442457\n",
       "ndc_cat54_Sum        7.552092\n",
       "DIAG_FLAG69_Sum      5.673218\n",
       "DIAG_FLAG5_Sum       5.406727\n",
       "REVCODE_FLAG10_Sum   4.901706\n",
       "REVCODE_FLAG19_Sum   4.670333\n",
       "ndc_cat61_Sum        3.886519\n",
       "DIAG_FLAG31_Sum      3.869930\n",
       "DIAG_FLAG71_Sum      3.583528\n",
       "ndc_cat58_Sum        3.453970\n",
       "ndc_cat98_Sum        2.951269\n",
       "ndc_cat87_Sum        2.894050\n",
       "ndc_cat30_Sum        2.370636\n",
       "ndc_cat43_Sum        2.337440\n",
       "ndc_cat13_Sum        1.970817\n",
       "ndc_cat59_Sum        1.880306\n",
       "ndc_cat6_Sum         1.725292\n",
       "age                  1.521835\n",
       "ndc_cat45_Sum        1.355579\n",
       "ndc_cat76_Sum        1.274270\n",
       "ndc_cat15_Sum        1.273675\n",
       "REVCODE_FLAG4_Sum    1.139761\n",
       "ndc_cat66_Sum        1.059264\n",
       "CPT_FLAG46_Sum       0.917191\n",
       "DIAG_FLAG86_Sum      0.887100\n",
       "ndc_cat42_Sum        0.746189\n",
       "CPT_FLAG14_Sum       0.661754\n",
       "ndc_cat67_Sum        0.607018\n",
       "ndc_cat40_Sum        0.592419\n",
       "ndc_cat1_Sum         0.505068\n",
       "REVCODE_FLAG62_Sum   0.494281\n",
       "Gender_M             0.486453\n",
       "ndc_cat11_Sum        0.472664\n",
       "DIAG_FLAG51_Sum      0.355996\n",
       "ndc_cat24_Sum        0.211333\n",
       "REVCODE_FLAG22_Sum   0.205492\n",
       "ndc_cat77_Sum        0.132837\n",
       "ndc_cat31_Sum        0.090761\n",
       "REVCODE_FLAG24_Sum   0.085903\n",
       "ndc_cat21_Sum       -0.116828\n",
       "ndc_cat63_Sum       -0.270736\n",
       "ndc_cat37_Sum       -0.470340\n",
       "DIAG_FLAG41_Sum     -0.509362\n",
       "ndc_cat96_Sum       -0.728530\n",
       "DIAG_FLAG11_Sum     -0.867685\n",
       "DIAG_FLAG49_Sum     -1.249089\n",
       "ndc_cat9_Sum        -1.340142\n",
       "ndc_cat34_Sum       -1.846452\n",
       "CPT_FLAG7_Sum       -2.373370\n",
       "DIAG_FLAG50_Sum     -2.383799\n",
       "DIAG_FLAG10_Sum     -3.181193\n",
       "ndc_cat19_Sum       -3.636398\n",
       "ndc_cat72_Sum       -3.640795\n",
       "CPT_FLAG5_Sum       -4.231993\n",
       "DIAG_FLAG55_Sum     -4.962752\n",
       "CPT_FLAG48_Sum     -13.397140\n",
       "CPT_FLAG4_Sum      -13.955133\n",
       "CPT_FLAG6_Sum      -24.699813\n",
       "DIAG_FLAG9_Sum     -28.111479"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp = pd.DataFrame(pipe.named_steps['clf'].base_estimator.coef_, columns=X_test.columns).T.sort_values(by=0, ascending=False)\n",
    "imp[imp[0] != 0.0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Do 3-Fold Cross Validation of LASSO - PN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed:   13.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARN: NestedCV.best_params_ set to None\n",
      "WARN: NestedCV.best_idxs_ set to None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.32460658700734701, 0.32100177981184846, 0.31133339831375795)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nested_lasso = NestedCV(pipe, scoring=FrankenScorer('assumed_f1beta10'), cv=3, random_state=None, use_same_random_state=False)\n",
    "nested_lasso.score(X_train.values, y_train.values, n_jobs=-1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "assumed_brier                                                       0.593557\n",
       "assumed_brier_neg                                                   0.596055\n",
       "assumed_f1                                                        0.00951951\n",
       "assumed_f1beta10                                                    0.318981\n",
       "confusion_matrix_lab       [[193.333333333, 94.0], [39.3333333333, 517.33...\n",
       "confusion_matrix_un        [[9331.0, 107631.0], [39.3333333333, 517.33333...\n",
       "fn_confusion_matrix_lab                                              39.3333\n",
       "fn_confusion_matrix_un                                               39.3333\n",
       "fp_confusion_matrix_lab                                                   94\n",
       "fp_confusion_matrix_un                                                107631\n",
       "labeled_acc                                                         0.842003\n",
       "labeled_avg_prec                                                    0.911138\n",
       "labeled_brier                                                       0.116227\n",
       "labeled_brier_neg                                                   0.208445\n",
       "labeled_brier_pos                                                  0.0686233\n",
       "labeled_f1                                                          0.885875\n",
       "labeled_prec                                                        0.846335\n",
       "labeled_recall                                                       0.92933\n",
       "labeled_roc_auc                                                     0.801069\n",
       "pr_one_unlabeled                                                    0.921683\n",
       "pu_mix_assumed_f1beta10                                              32.8369\n",
       "pu_score                                                            0.938834\n",
       "tn_confusion_matrix_lab                                              193.333\n",
       "tn_confusion_matrix_un                                                  9331\n",
       "tp_confusion_matrix_lab                                              517.333\n",
       "tp_confusion_matrix_un                                               517.333\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_scores_from_nested(nested_lasso.test_score_datas_).mean().sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3-Fold Nested Cross random search for pipline with PNU wrapper which will _undersample_ the unlabeled data by a massive amount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_search = {'clf__base_estimator__C': sp.stats.expon(scale=4), \n",
    "               'clf__num_unlabeled': sp.stats.randint(low=1000, high=10000),\n",
    "               'clf__base_estimator__class_weight': [None, 'balanced'],\n",
    "               'clf__base_estimator__penalty': ['l1','l2'],\n",
    "               'clf__pu_learning': [True, False]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jsearch = JRandomSearchCV(pipe, param_search, n_iter=20, \n",
    "                            scoring=FrankenScorer('assumed_f1beta10'), n_jobs=-1, cv=5, verbose=1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nested_cross = NestedCV(jsearch, scoring=FrankenScorer('assumed_f1beta10'), cv=3, random_state=731)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed:  8.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.56329637821774114, 0.54711689168751887, 0.54173371291826544)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nested_cross.score(X_train.values, y_train.values, n_jobs=-1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "assumed_brier                                                       0.129326\n",
       "assumed_brier_neg                                                   0.129123\n",
       "assumed_f1                                                         0.0444195\n",
       "assumed_f1beta10                                                    0.550716\n",
       "confusion_matrix_lab       [[175.0, 112.333333333], [158.333333333, 398.3...\n",
       "confusion_matrix_un        [[99981.0, 16981.0], [158.333333333, 398.33333...\n",
       "fn_confusion_matrix_lab                                              158.333\n",
       "fn_confusion_matrix_un                                               158.333\n",
       "fp_confusion_matrix_lab                                              112.333\n",
       "fp_confusion_matrix_un                                                 16981\n",
       "labeled_acc                                                         0.679287\n",
       "labeled_avg_prec                                                    0.841611\n",
       "labeled_brier                                                       0.205563\n",
       "labeled_brier_neg                                                   0.270436\n",
       "labeled_brier_pos                                                   0.172083\n",
       "labeled_f1                                                           0.74641\n",
       "labeled_prec                                                        0.780054\n",
       "labeled_recall                                                       0.71556\n",
       "labeled_roc_auc                                                     0.662277\n",
       "pr_one_unlabeled                                                    0.144579\n",
       "pu_mix_assumed_f1beta10                                               58.534\n",
       "pu_score                                                             3.46244\n",
       "tn_confusion_matrix_lab                                                  175\n",
       "tn_confusion_matrix_un                                                 99981\n",
       "tp_confusion_matrix_lab                                              398.333\n",
       "tp_confusion_matrix_un                                               398.333\n",
       "dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_scores_from_nested(nested_cross.test_score_datas_).mean().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "assumed_brier                0.001913\n",
       "assumed_brier_neg            0.001965\n",
       "assumed_f1                   0.000336\n",
       "assumed_f1beta10             0.011223\n",
       "fn_confusion_matrix_lab     10.785793\n",
       "fn_confusion_matrix_un      10.785793\n",
       "fp_confusion_matrix_lab     11.718931\n",
       "fp_confusion_matrix_un     522.054595\n",
       "labeled_acc                  0.026653\n",
       "labeled_avg_prec             0.014438\n",
       "labeled_brier                0.013491\n",
       "labeled_brier_neg            0.022241\n",
       "labeled_brier_pos            0.009056\n",
       "labeled_f1                   0.020772\n",
       "labeled_prec                 0.022466\n",
       "labeled_recall               0.019556\n",
       "labeled_roc_auc              0.030194\n",
       "pr_one_unlabeled             0.004574\n",
       "pu_mix_assumed_f1beta10      1.209920\n",
       "pu_score                     0.088974\n",
       "tn_confusion_matrix_lab     12.288206\n",
       "tn_confusion_matrix_un     521.230275\n",
       "tp_confusion_matrix_lab     11.150486\n",
       "tp_confusion_matrix_un      11.150486\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_scores_from_nested(nested_cross.test_score_datas_).std().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'clf__base_estimator__C': 3.9037130019710689,\n",
       "  'clf__base_estimator__class_weight': 'balanced',\n",
       "  'clf__base_estimator__penalty': 'l1',\n",
       "  'clf__num_unlabeled': 2610,\n",
       "  'clf__pu_learning': True},\n",
       " {'clf__base_estimator__C': 1.485616929764805,\n",
       "  'clf__base_estimator__class_weight': 'balanced',\n",
       "  'clf__base_estimator__penalty': 'l1',\n",
       "  'clf__num_unlabeled': 9623,\n",
       "  'clf__pu_learning': True},\n",
       " {'clf__base_estimator__C': 1.485616929764805,\n",
       "  'clf__base_estimator__class_weight': 'balanced',\n",
       "  'clf__base_estimator__penalty': 'l1',\n",
       "  'clf__num_unlabeled': 9623,\n",
       "  'clf__pu_learning': True}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[est.best_params_ for est in nested_cross.estimators_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}