Evaluating a Classification Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the data into a Pandas DataFrame\n",
    "import pandas as pd\n",
    "\n",
    "col_names = ['pregnant', 'glucose', 'bp', 'skin', 'insulin', 'bmi', 'pedigree', 'age','label']\n",
    "pima = pd.read_csv('pima-indians-diabetes.csv', header=None, names=col_names, skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnant</th>\n",
       "      <th>glucose</th>\n",
       "      <th>bp</th>\n",
       "      <th>skin</th>\n",
       "      <th>insulin</th>\n",
       "      <th>bmi</th>\n",
       "      <th>pedigree</th>\n",
       "      <th>age</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnant  glucose  bp  skin  insulin   bmi  pedigree  age  label\n",
       "0         6      148  72    35        0  33.6     0.627   50      1\n",
       "1         1       85  66    29        0  26.6     0.351   31      0\n",
       "2         8      183  64     0        0  23.3     0.672   32      1\n",
       "3         1       89  66    23       94  28.1     0.167   21      0\n",
       "4         0      137  40    35      168  43.1     2.288   33      1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 5 rows of data from the dataframe\n",
    "pima.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "pregnant    768 non-null int64\n",
      "glucose     768 non-null int64\n",
      "bp          768 non-null int64\n",
      "skin        768 non-null int64\n",
      "insulin     768 non-null int64\n",
      "bmi         768 non-null float64\n",
      "pedigree    768 non-null float64\n",
      "age         768 non-null int64\n",
      "label       768 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "pima.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- label\n",
    " - 1: diabetes\n",
    " - 0: diabetes\n",
    "- pregnant\n",
    " - number of times pregnant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question: Can we predict the diabetes status of a patient given their health measurements?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define X and y\n",
    "feature_cols = ['pregnant', 'insulin', 'bmi', 'age']\n",
    "\n",
    "# X is a matrix, hence we use [] to access the features we want in feature_cols\n",
    "X = pima[feature_cols]\n",
    "\n",
    "# y is a vector, hence we use dot to access 'label'\n",
    "y = pima.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split X and y into training and testing sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train a logistic regression model on the training set\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# instantiate model\n",
    "logreg = LogisticRegression()\n",
    "\n",
    "# fit model\n",
    "logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make class predictions for the testing set\n",
    "y_pred_class = logreg.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification accuracy:** percentage of correct predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6927083333333334\n"
     ]
    }
   ],
   "source": [
    "# calculate accuracy\n",
    "from sklearn import metrics\n",
    "print(metrics.accuracy_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classification accuracy is 69%\n",
    "\n",
    "**Null accuracy:** accuracy that could be achieved by always predicting the most frequent class\n",
    "\n",
    "- We must always compare with this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    130\n",
       "1     62\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the class distribution of the testing set (using a Pandas Series method)\n",
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3229166666666667"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate the percentage of ones\n",
    "# because y_test only contains ones and zeros, we can simply calculate the mean = percentage of ones\n",
    "y_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6770833333333333"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate the percentage of zeros\n",
    "1 - y_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6770833333333333"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate null accuracy in a single line of code\n",
    "# only for binary classification problems coded as 0/1\n",
    "max(y_test.mean(), 1 - y_test.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means that a dumb model that always predicts 0 would be right 68% of the time\n",
    "\n",
    "- This shows how classification accuracy is not that good as it's close to a dumb model\n",
    "- It's a good way to know the minimum we should achieve with our models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.677083\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate null accuracy (for multi-class classification problems)\n",
    "y_test.value_counts().head(1) / len(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing the **true** and **predicted** response values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True: [1 0 0 1 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 1 1 0 0 0]\n",
      "False: [0 0 0 0 0 0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "# print the first 25 true and predicted responses\n",
    "print('True:', y_test.values[0:25])\n",
    "print('False:', y_pred_class[0:25])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Conclusion:**\n",
    "\n",
    "- Classification accuracy is the **easiest classification metric to understand**\n",
    "- But, it does not tell you the **underlying distribution** of response values\n",
    " - We examine by calculating the null accuracy\n",
    "- And, it does not tell you what **\"types\" of errors** your classifier is making"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix\n",
    "\n",
    "Table that describes the performance of a classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[118  12]\n",
      " [ 47  15]]\n"
     ]
    }
   ],
   "source": [
    "# IMPORTANT: first argument is true values, second argument is predicted values\n",
    "# this produces a 2x2 numpy array (matrix)\n",
    "print(metrics.confusion_matrix(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Every observation in the testing set is represented in **exactly one box**\n",
    "- It's a 2x2 matrix because there are **2 response classes**\n",
    "- The format shown here is **not** universal\n",
    " - Take attention to the format when interpreting a confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Basic terminology**\n",
    "\n",
    "- **True Positives (TP):** we correctly predicted that they do have diabetes\n",
    " - 15\n",
    "- **True Negatives (TN):** we correctly predicted that they don't have diabetes\n",
    " - 118\n",
    "- **False Positives (FP):** we incorrectly predicted that they do have diabetes (a \"Type I error\")\n",
    " - 12\n",
    " - Falsely predict positive\n",
    " - Type I error\n",
    "- **False Negatives (FN):** we incorrectly predicted that they don't have diabetes (a \"Type II error\")\n",
    " - 47\n",
    " - Falsely predict negative\n",
    " - Type II error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 0: negative class\n",
    "- 1: positive class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[118  12]\n",
      " [ 47  15]]\n"
     ]
    }
   ],
   "source": [
    "# save confusion matrix and slice into four pieces\n",
    "confusion = metrics.confusion_matrix(y_test, y_pred_class)\n",
    "print(confusion)\n",
    "#[row, column]\n",
    "TP = confusion[1, 1]\n",
    "TN = confusion[0, 0]\n",
    "FP = confusion[0, 1]\n",
    "FN = confusion[1, 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics computed from a confusion matrix\n",
    "\n",
    "**Classification Accuracy:** Overall, how often is the classifier correct?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6927083333333334\n",
      "0.6927083333333334\n"
     ]
    }
   ],
   "source": [
    "# use float to perform true division, not integer division\n",
    "print((TP + TN) / float(TP + TN + FP + FN))\n",
    "print(metrics.accuracy_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification Error:** Overall, how often is the classifier incorrect?\n",
    "\n",
    " - Also known as \"Misclassification Rate\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3072916666666667\n",
      "0.30729166666666663\n"
     ]
    }
   ],
   "source": [
    "classification_error = (FP + FN) / float(TP + TN + FP + FN)\n",
    "\n",
    "print(classification_error)\n",
    "print(1 - metrics.accuracy_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Sensitivity:** When the actual value is positive, how often is the prediction correct?\n",
    "\n",
    "- Something we want to maximize\n",
    "- How \"sensitive\" is the classifier to detecting positive instances?\n",
    "- Also known as \"True Positive Rate\" or \"Recall\"\n",
    "- TP / all positive\n",
    " - all positive = TP + FN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.24193548387096775\n",
      "0.24193548387096775\n"
     ]
    }
   ],
   "source": [
    "sensitivity = TP / float(FN + TP)\n",
    "\n",
    "print(sensitivity)\n",
    "print(metrics.recall_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Specificity:** When the actual value is negative, how often is the prediction correct?\n",
    "\n",
    "- Something we want to maximize\n",
    "- How \"specific\" (or \"selective\") is the classifier in predicting positive instances?\n",
    "- TN / all negative\n",
    " - all negative = TN + FP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9076923076923077\n"
     ]
    }
   ],
   "source": [
    "specificity = TN / (TN + FP)\n",
    "\n",
    "print(specificity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our classifier\n",
    "\n",
    "- Highly specific\n",
    "- Not sensitive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**False Positive Rate:** When the actual value is negative, how often is the prediction incorrect?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.09230769230769231\n",
      "0.09230769230769231\n"
     ]
    }
   ],
   "source": [
    "false_positive_rate = FP / float(TN + FP)\n",
    "\n",
    "print(false_positive_rate)\n",
    "print(1 - specificity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Precision:** When a positive value is predicted, how often is the prediction correct?\n",
    "\n",
    "- How \"precise\" is the classifier when predicting positive instances?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5555555555555556\n",
      "0.5555555555555556\n"
     ]
    }
   ],
   "source": [
    "precision = TP / float(TP + FP)\n",
    "\n",
    "print(precision)\n",
    "print(metrics.precision_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Conclusion:**\n",
    "\n",
    "- Confusion matrix gives you a **more complete picture** of how your classifier is performing\n",
    "- Also allows you to compute various **classification metrics**, and these metrics can guide your model selection\n",
    "**Which metrics should you focus on?**\n",
    "\n",
    "- Choice of metric depends on your **business objective**\n",
    " - Identify if FP or FN is more important to reduce\n",
    " - Choose metric with relevant variable (FP or FN in the equation)\n",
    "- Spam filter (positive class is \"spam\"):\n",
    " - Optimize for **precision or specificity**\n",
    "  - precision\n",
    "   - false positive as variable\n",
    "  - specificity\n",
    "   - false positive as variable\n",
    " - Because false negatives (spam goes to the inbox) are more acceptable than false positives (non-spam is caught by the spam filter)\n",
    "- **Fraudulent transaction detector** (positive class is \"fraud\"):\n",
    " - Optimize for sensitivity\n",
    "   - FN as a variable\n",
    " - Because false positives (normal transactions that are flagged as possible fraud) are more acceptable than false negatives (fraudulent transactions that are not detected)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adjusting the classification threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 1, 0, 1], dtype=int64)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 10 predicted responses\n",
    "# 1D array (vector) of binary values (0, 1)\n",
    "logreg.predict(X_test)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.63247571, 0.36752429],\n",
       "       [0.71643656, 0.28356344],\n",
       "       [0.71104114, 0.28895886],\n",
       "       [0.5858938 , 0.4141062 ],\n",
       "       [0.84103973, 0.15896027],\n",
       "       [0.82934844, 0.17065156],\n",
       "       [0.50110974, 0.49889026],\n",
       "       [0.48658459, 0.51341541],\n",
       "       [0.72321388, 0.27678612],\n",
       "       [0.32810562, 0.67189438]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 10 predicted probabilities of class membership\n",
    "logreg.predict_proba(X_test)[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Row: observation\n",
    " - Each row, numbers sum to 1\n",
    "- Column: class\n",
    " - 2 response classes there 2 columns\n",
    "   - column 0: predicted probability that each observation is a member of class 0\n",
    "   - column 1: predicted probability that each observation is a member of class 1\n",
    "- Importance of predicted probabilities\n",
    " - We can rank observations by probability of diabetes\n",
    "   - Prioritize contacting those with a higher probability\n",
    "- predict_proba process\n",
    " 1. Predicts the probabilities\n",
    " 2. Choose the class with the highest probability\n",
    "- There is a 0.5 classification threshold\n",
    " 1. Class 1 is predicted if probability > 0.5\n",
    " 2. Class 0 is predicted if probability < 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.36752429, 0.28356344, 0.28895886, 0.4141062 , 0.15896027,\n",
       "       0.17065156, 0.49889026, 0.51341541, 0.27678612, 0.67189438])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 10 predicted probabilities for class 1\n",
    "logreg.predict_proba(X_test)[0:10, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store the predicted probabilities for class 1\n",
    "y_pred_prob = logreg.predict_proba(X_test)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allow plots to appear in the notebook\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# adjust the font size \n",
    "plt.rcParams['font.size'] = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Frequency')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHXBJREFUeJzt3XmcHVWd9/HPNwlLWAMkIIalIYT9UcCwic4gIi8E2UYQGBBQEAFnABEGRAbhUWdgkEF9mEd2WWWVJQMqQiQEGLaw7xIhQGQLa9jX3/xxTidVN7e7q5u+t7o73/frdV9dy6mqX527/O451feUIgIzM7NOw+oOwMzMBhYnBjMzK3FiMDOzEicGMzMrcWIwM7MSJwYzMytxYhiEJD0kadO646iTpB0kPSPpTUnr1nD8yZL2ydO7SfpTG47ZISkkjWj1sfLxQtIqfdx2uqTNu1j3RUmPNSsr6UhJZ3Sz37bU9bzOiWGAafaGkrSXpJs75yNirYiY3MN+2vohUoOfA/8UEYtExD11BhIRF0TEFj2Vk3SMpPPbEdNAFhE3RcRqXaz7t4joTLhzvYar1rV9Mk4M1icDIOGsCDzUHzsaAOfSdvPiOVt1TgyDUEPTewNJUyXNkvSCpP/Mxabkv6/l7paNJQ2TdJSkpyS9KOlcSYsX9rtHXveypH9tOM4xki6TdL6kWcBe+di3SnpN0nOSTpY0f2F/IekASY9LekPSTySNy9vMknRJsXzDOTaNVdICkt4EhgP3SfprF9uHpAMlPSHpJUknSBqW1+0l6RZJJ0l6BTgmL/+2pEckvSrpWkkrFvb3FUmPSnpd0smACutKLTpJa0m6TtIr+Tk5UtKWwJHAzvn5uC+XXVzSmbn+/ibpp5KG53XDJf08x/8EsHWF18UPJT2cz+E3khbM6zaVNEPS4ZKeB36Tl39H0rQc60RJn27Y7VZd1OE4SX/Or5WXJF0gaVTDtut3F0sX51BsVTV7DTfW9eqFun5M0jcK67bKx38j1+2h3dWfFUSEHwPoAUwHNm9Ythdwc7MywK3AN/P0IsBGeboDCGBEYbtvA9OAlXPZy4Hz8ro1gTeBLwDzk7pqPigc55g8vz3pC8VI4HPARsCIfLxHgIMLxwtgIrAYsBbwHjApH39x4GFgzy7qoctYC/tepZt6DOAGYElgBeAvwD6F+vwQ+Occ+8h8XtOANfKyo4D/yeVHA7OAHYH5gO/n7fdpfH6ARYHngB8AC+b5DQt1eH5DnFcCpwILA0sDdwDfzev2Ax4Fls/ncUPjc9rktfNgofwtwE/zuk1zzMcDC+Rz3gx4CVgvL/t/wJSKdbgK8JW83RjSh/gvehHLjC5ez7PriOav4WJdLww8A3wrP2fr5fNZK69/Dvhinl4CWK/u9/dgedQegB8NT0h6k7wJvFZ4vE3XiWEKcCwwumE/zd5Uk4ADCvOrkT7sRwBHAxcW1i0EvN/whp3SQ+wHA1cU5gPYpDB/F3B4Yf7E4odJw766jLWw754Sw5aF+QOASXl6L+DphvJ/APYuzA/L9b4isAdwW2GdgBk0Twy7Avd0EdPsD708vwwpWY4sLNsVuCFP/xnYr7Bui8bntMlrp1h+K+CveXrT/HwuWFh/JvAfhflFch139FSHTY69ffG8K8TSH4lhZ+CmhjhOBX6cp58Gvgss1q7371B5uCtpYNo+IkZ1PkhvyK7sDawKPCrpTklf66bsp4GnCvNPkZLCMnndM50rIuJt4OWG7Z8pzkhaVdLVkp7P3Uv/Rvp2XfRCYfqdJvOL9CHWqorxPpX32WwdpATwy9wt9hrwCikBjGXuuokm23daHmjavdXEiqQWyHOF455KajnQeFzK9dGV7s55ZkS8W5gv1XFEvEl6zsf2tD9JS0u6KHfRzALOZ+7nvrtY+sOKwIaddZfrbzfgU3n910kJ6SlJN0rauJ+PP2Q5MQxyEfF4ROxK+jA5HrhM0sKkb1qNniW9mTqtQOpeeIHU7F6uc4WkkcBSjYdrmP81qatjfEQsRupDF/2ju1irWr5h+2cL843n8gypC2dU4TEyIv6HVDez9yVJDftu3M+4LtY1O+Z7pNZe5zEXi4i18vrScfM59KQ351yq4/y6WQr4W4X9/Xve32fyc787cz/33cVSRU9DPz8D3NjwnC0SEfsDRMSdEbEd6b1xJXBJL48/z3JiGOQk7S5pTER8TOp2AvgImAl8TOqj73Qh8H1JK0lahPQN/+KI+BC4DNhG0ueVLggfS88f8ouS+t7flLQ6sH+/nVj3sVZ1mKQlJC0PHARc3E3ZU4AfSloLZl8U3imvuwZYS9I/KP03z4HM+Vba6GrgU5IOVrpQvqikDfO6F4COzgu4EfEc8CfgREmLKV1wHyfp73P5S4ADJS0naQngiArn/L1cfklSou7unH8LfEvSOpIWINXx7RExvVCmqzpclNzlKWkscNgnjKWZZq/hoquBVSV9U9J8+bG+pDUkza/0m4fFI+ID0uv0o14ef57lxDD4bQk8pPSfOr8EdomId3NX0M+AW3IzeyPgLOA80nWJJ4F3SRdgiYiH8vRFpG+qbwAvkr7RduVQ4B9z2dPp/Ru/O13G2gtXka5r3Ev6cD+zq4IRcQWpxXVR7hp5EPhqXvcSsBNwHKmrZTzpYmqz/bxBuii7DfA88Djwpbz60vz3ZUl35+k9SBf7HwZeJSXoZfO604FrgfuAu0kX4HvyW1KyeSI/ftrNOU8C/hX4Hek5Hwfs0lCsqzo8lnSx9/W8vFlslWPpIr5mr+Hi+jdI1112IbVGnmfOxXWAbwLT8/O5H6lVYxUoX6QxK8nf0l8jdRM9WXc8vSUpSLFPqzuWdpE0nXRB/Pq6Y7HBzS0Gm03SNpIWyn3NPwceIP3HiJnNQ5wYrGg7UpP8WVJ3yS7hJqXZPMddSWZmVuIWg5mZlQyKgbRGjx4dHR0ddYdhZjao3HXXXS9FxJjebjcoEkNHRwdTp06tOwwzs0FFUpVfy8/FXUlmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVjIofvlsPes44ppajz/9uK1rPb6Z9R+3GMzMrMSJwczMSpwYzMysxInBzMxKnBjMzKzEicHMzEqcGMzMrMSJwczMSpwYzMysxInBzMxKnBjMzKzEicHMzEo8iF4/qHsAOzOz/uQWg5mZlTgxmJlZiRODmZmVODGYmVmJE4OZmZU4MZiZWYkTg5mZlTgxmJlZiRODmZmVODGYmVlJyxODpOGS7pF0dZ5fSdLtkh6XdLGk+Vsdg5mZVdeOFsNBwCOF+eOBkyJiPPAqsHcbYjAzs4pamhgkLQdsDZyR5wVsBlyWi5wDbN/KGMzMrHda3WL4BfAvwMd5fingtYj4MM/PAMY221DSvpKmSpo6c+bMFodpZmadWpYYJH0NeDEi7ioublI0mm0fEadFxISImDBmzJiWxGhmZnNr5f0YNgG2lbQVsCCwGKkFMUrSiNxqWA54toUxmJlZL7WsxRARP4yI5SKiA9gF+HNE7AbcAOyYi+0JXNWqGMzMrPfq+B3D4cAhkqaRrjmcWUMMZmbWhbbc2jMiJgOT8/QTwAbtOK6ZmfWef/lsZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW0pZB9MzaoeOIa2o9/vTjtq71+Gb9xS0GMzMrcWIwM7MSJwYzMytxYjAzsxInBjMzK3FiMDOzEicGMzMrcWIwM7MSJwYzMytxYjAzsxInBjMzK3FiMDOzEicGMzMrcWIwM7MSJwYzMytxYjAzsxInBjMzK/Ed3Kxf1H33NDPrP24xmJlZiRODmZmVODGYmVmJE4OZmZU4MZiZWUnLEoOkBSXdIek+SQ9JOjYvX0nS7ZIel3SxpPlbFYOZmfVeK1sM7wGbRcRngXWALSVtBBwPnBQR44FXgb1bGIOZmfVSyxJDJG/m2fnyI4DNgMvy8nOA7VsVg5mZ9V5LrzFIGi7pXuBF4Drgr8BrEfFhLjIDGNvFtvtKmipp6syZM1sZppmZFbQ0MUTERxGxDrAcsAGwRrNiXWx7WkRMiIgJY8aMaWWYZmZWUCkxSFr7kxwkIl4DJgMbAaMkdQ7FsRzw7CfZt5mZ9a+qLYZT8n8YHSBpVJUNJI3pLCtpJLA58AhwA7BjLrYncFUvYzYzsxaqlBgi4gvAbsDywFRJv5X0lR42Wxa4QdL9wJ3AdRFxNXA4cIikacBSwJl9jt7MzPpd5dFVI+JxSUcBU4FfAetKEnBkRFzepPz9wLpNlj9But5gZmYDUNVrDJ+RdBKpK2gzYJuIWCNPn9TC+MzMrM2qthhOBk4ntQ7e6VwYEc/mVoSZmQ0RVRPDVsA7EfERgKRhwIIR8XZEnNey6MzMrO2q/lfS9cDIwvxCeZmZmQ0xVRPDgoXhLcjTC7UmJDMzq1PVxPCWpPU6ZyR9Dninm/JmZjZIVb3GcDBwqaTOXykvC+zcmpDMzKxOlRJDRNwpaXVgNUDAoxHxQUsjMzOzWlT+gRuwPtCRt1lXEhFxbkuiMjOz2lRKDJLOA8YB9wIf5cUBODGYmQ0xVVsME4A1I6LpENlmZjZ0VP2vpAeBT7UyEDMzGxiqthhGAw9LuoN0L2cAImLblkRlZma1qZoYjmllEGZmNnBU/XfVGyWtCIyPiOslLQQMb21oZmZWh6rDbn8HuAw4NS8aC1zZqqDMzKw+VS8+fw/YBJgF6aY9wNKtCsrMzOpTNTG8FxHvd85IGkH6HYOZmQ0xVRPDjZKOBEbmez1fCvx368IyM7O6VE0MRwAzgQeA7wK/B3znNjOzIajqfyV9TLq15+mtDcfMzOpWdaykJ2lyTSEiVu73iMzMrFa9GSup04LATsCS/R+OmZnVrdI1hoh4ufD4W0T8AtisxbGZmVkNqnYlrVeYHUZqQSzakojMzKxWVbuSTixMfwhMB77R79GYmVntqv5X0pdaHYiZmQ0MVbuSDulufUT8Z/+EY2ZmdevNfyWtD0zM89sAU4BnWhGUmZnVpzc36lkvIt4AkHQMcGlE7NOqwMzMrB5Vh8RYAXi/MP8+0NHv0ZiZWe2qthjOA+6QdAXpF9A7AOe2LCozM6tN1f9K+pmkPwBfzIu+FRH3tC4sMzOrS9WuJICFgFkR8UtghqSVWhSTmZnVqOqtPX8MHA78MC+aDzi/VUGZmVl9qrYYdgC2Bd4CiIhn8ZAYZmZDUtXE8H5EBHnobUkL97SBpOUl3SDpEUkPSTooL19S0nWSHs9/l+h7+GZm1t+qJoZLJJ0KjJL0HeB6er5pz4fADyJiDWAj4HuS1iTdDW5SRIwHJuV5MzMbIKr+V9LP872eZwGrAUdHxHU9bPMc8FyefkPSI8BYYDtg01zsHGAy6fqFmZkNAD0mBknDgWsjYnOg22TQzT46gHWB24FlctIgIp6TtHQX2+wL7Auwwgor9OWwZmbWBz12JUXER8DbkhbvywEkLQL8Djg4ImZV3S4iTouICRExYcyYMX05tJmZ9UHVXz6/Czwg6TryfyYBRMSB3W0kaT5SUrggIi7Pi1+QtGxuLSwLvNiHuM3MrEWqJoZr8qMySQLOBB5pGJZ7IrAncFz+e1Vv9mtmZq3VbWKQtEJEPB0R5/Rh35sA3yS1NO7Ny44kJYRLJO0NPA3s1Id9m5lZi/TUYrgSWA9A0u8i4utVdxwRNwPqYvWXq+7HzMzaq6eLz8UP9pVbGYiZmQ0MPSWG6GLazMyGqJ66kj4raRap5TAyT5PnIyIWa2l0ZmbWdt0mhogY3q5AzMxsYOjN/RjMzGwe4MRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlI+oOwMz6T8cR19R6/OnHbV3r8a1/uMVgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJS1LDJLOkvSipAcLy5aUdJ2kx/PfJVp1fDMz65tWthjOBrZsWHYEMCkixgOT8ryZmQ0gLUsMETEFeKVh8XbAOXn6HGD7Vh3fzMz6pt3XGJaJiOcA8t+luyooaV9JUyVNnTlzZtsCNDOb1w3Yi88RcVpETIiICWPGjKk7HDOzeUa7E8MLkpYFyH9fbPPxzcysB+1ODBOBPfP0nsBVbT6+mZn1oGWD6Em6ENgUGC1pBvBj4DjgEkl7A08DO/XHseoeOMwM/Dq0oaNliSEidu1i1ZdbdUwzM/vkBuzFZzMzq4cTg5mZlTgxmJlZiRODmZmVODGYmVmJE4OZmZU4MZiZWYkTg5mZlTgxmJlZiRODmZmVODGYmVmJE4OZmZU4MZiZWYkTg5mZlTgxmJlZiRODmZmVtOxGPWY27xkId7GbftzWdYcw6LnFYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJU4MZmZW4sRgZmYlTgxmZlbixGBmZiVODGZmVuLEYGZmJbUkBklbSnpM0jRJR9QRg5mZNdf2xCBpOPBfwFeBNYFdJa3Z7jjMzKy5OloMGwDTIuKJiHgfuAjYroY4zMysiRE1HHMs8ExhfgawYWMhSfsC++bZ9yQ92IbYBoPRwEt1BzFAuC7mcF1kOt51UbBaXzaqIzGoybKYa0HEacBpAJKmRsSEVgc2GLgu5nBdzOG6mMN1MYekqX3Zro6upBnA8oX55YBna4jDzMyaqCMx3AmMl7SSpPmBXYCJNcRhZmZNtL0rKSI+lPRPwLXAcOCsiHioh81Oa31kg4brYg7XxRyuizlcF3P0qS4UMVf3vpmZzcP8y2czMytxYjAzs5IBlRh6GipD0gKSLs7rb5fU0f4oW69CPRwi6WFJ90uaJGnFOuJsh6rDp0jaUVJIGrL/plilLiR9I782HpL023bH2C4V3iMrSLpB0j35fbJVHXG2g6SzJL3Y1W+9lPwq19X9ktbrcacRMSAepAvRfwVWBuYH7gPWbChzAHBKnt4FuLjuuGuqhy8BC+Xp/YdiPVSti1xuUWAKcBswoe64a3xdjAfuAZbI80vXHXeNdXEasH+eXhOYXnfcLayPvwPWAx7sYv1WwB9IvyHbCLi9p30OpBZDlaEytgPOydOXAV+W1OwHc4NZj/UQETdExNt59jbSb0GGoqrDp/wE+A/g3XYG12ZV6uI7wH9FxKsAEfFim2Nslyp1EcBieXpxhvBvpSJiCvBKN0W2A86N5DZglKRlu9vnQEoMzYbKGNtVmYj4EHgdWKot0bVPlXoo2pv0bWAo6rEuJK0LLB8RV7czsBpUeV2sCqwq6RZJt0nasm3RtVeVujgG2F3SDOD3wD+3J7QBqbefKbUMidGVKkNlVBpOY5CrfI6SdgcmAH/f0ojq021dSBoGnATs1a6AalTldTGC1J20KakVeZOktSPitRbH1m5V6mJX4OyIOFHSxsB5uS4+bn14A06vPzcHUouhylAZs8tIGkFqInbXhBqMKg0ZImlz4EfAthHxXptia7ee6mJRYG1gsqTppP7TiUP0AnTV98dVEfFBRDwJPEZKFENNlbrYG7gEICJuBRYkDTQ4L+r1MEQDKTFUGSpjIrBnnt4R+HPkqytDSI/1kLtPTiUlhaHajww91EVEvB4RoyOiIyI6SNdbto2IPg0cNsBVeX9cSfrHBCSNJnUtPdHWKNujSl08DXwZQNIapMQws61RDhwTgT3yfydtBLweEc91t8GA6UqKLobKkPR/gakRMRE4k9QknEZqKexSX8StUbEeTgAWAS7N196fjohtawu6RSrWxTyhYl1cC2wh6WHgI+CwiHi5vqhbo2Jd/AA4XdL3Sd0mew3BL5EASLqQ1H04Ol9T+TEwH0BEnEK6xrIVMA14G/hWj/sconVlZmZ9NJC6kszMbABwYjAzsxInBjMzK3FiMDOzEicGMzMrcWKYx0n6SNK9kh6UdKmkhT7BvjaVdHWe3raH0VBHSTqgD8c4RtKhfY2xm/3Ojr0X20zPvxdoXL6fpD3y9NmSdszTZ0haM08f2R9x530dKOkRSRf0UG5y54//JP1e0qgeyr/Zyzi27zw/G9ycGOydiFgnItYG3gf2K67MP4rp9eskIiZGxHHdFBlFGi23bfKv5VsuIk6JiHObLN8nIh7Os/2WGEj1uFVE7FZ1g4jYqgVDZWxPGsnUBjknBiu6CVhFUkf+Bvr/gbuB5SVtIelWSXfnlsUiMHtc/Ecl3Qz8Q+eOJO0l6eQ8vYykKyTdlx+fB44DxuXWygm53GGS7sxjxh9b2NePlMbevx5YrVng+Zv5KZJukvQXSV8rxHGppP8G/pQT3Qm5hfSApJ0Lu1ksx/lw3tewvI9fS5qqdI+DYxsOfZikO/JjlVy+aaum8xu7pOOAkfncL5D0E0kHFcr9TNKBTbY/JMf9oKSD87JTSMNPT8w/5iqWHynpolyfFwMjC+tmt3YkXSnprnx++zbs48T8nE+SNCYvGyfpj3mbmyStnp/TbYET8nmNa1Yub79TPof7JE1p9nxazeoeS9yPeh/Am/nvCOAq0v0dOoCPgY3yutGk+x0snOcPB44mDTPwDGk8HpHGprk6l9kLODlPXwwcnKeHk8a46qAwfjywBWkMfZG+sFxNGmf+c8ADwEKkYZSnAYc2OY+zgT/mbceTxodZMMcxA1gyl/s6cF2OYxnS0AnLkn45+i7pQ3Z4LrNj3mbJQuyTgc/k+enAj/L0HoVzP6YzxhxX534mk+8X0VnveboDuDtPDyPda2CphvPrrIeFSb96fwhYtxDH6CZ1cgjpV8EAnwE+LBx/9jaF8xsJPNh5bNIvhnfL00cXns9JwPg8vSFpaJrSufZQ7gFgbJ4eVfd7wI+5HwNmSAyrzUhJ9+bpm0jDjnwaeCrS2O2QBqdbE7hFaQiO+YFbgdWBJyPicQBJ5wOlb5zZZqQPTiLiI+B1SUs0lNkiP+7J84uQPuAXBa6IfP8JSd0Ng3FJpNEzH5f0RI4P4LqI6Bxs8QvAhTmOFyTdCKwPzALuiIgn8nEuzGUvA76Rv0mPICWRNYH78/4uLPw9qZvYuhQR0yW9rDQG1jLAPTH3UBZfINXDWzm+y4EvMqe+mvk74Ff5GPdLur+LcgdK2iFPL0+q95dJXw4uzsvPBy7PLcXPM2c4FoAFGnfYQ7lbgLMlXQJc3k38VhMnBnsnItYpLshv5LeKi0gfrrs2lFuH/hv2XMC/R8SpDcc4uBfHaCzXOd94LpW3l7QScCiwfkS8KulsUkuk2TafpC7OILVuPgWc1WR9X29I1W1MkjYFNgc2joi3JU2mfH6N+xoGvNb4mmmiy3IRsZ+kDYGtgXslrdMkEVqNfI3BqrgN2KTQh76QpFWBR4GVJI3L5XbtYvtJpC4qJA2XtBjwBqk10Ola4Nuac+1irKSlSV1YO+T+8kWBbbqJcydJw3I8K5OGnW40Bdg5xzGG9K36jrxuA6URO4cBOwM3k7qv3iK1cpYBvtqwv50Lf2/tJrZGH0iarzB/BbAlqfVybRdxb5/rfmFgB1ILrztTgN0AJK1N6k5qtDjwak4Kq5Nah52GkUYxBvhH4OaImAU8KWmnvF9J+mwuM/s57a6cpHERcXtEHA28RHlIaBsA3GKwHkXETEl7ARdK6uwOOCoi/pK7WK6R9BLpg3TtJrs4CDhN0t6kUT/3j4hble409iDwh4g4TGl45Ftzi+VNYPeIuDtfOL0XeIruPwwfA24kdcfsFxHvau47v14BbEy6T3AA/xIRz+cPxVtJF8X/D+lD9YqI+FjSPaQ+/SdI3SBFC0i6nfQh2lVibOY04H5Jd0fEbhHxvqQbSN+yP2osnOvhbOYksTMiortuJIBfA7/JXUj3FrYt+iOwXy7zGOlLQKe3gLUk3UW6W2JnEtwN+LWko0ijeF5Eqs+LSCOaHkhKKF2VO0FS53WpSXmZDSAeXdWGhPyheXVEXFZ3LH2RWyl3Azt1XrMxq4u7ksxqpvSjsGnAJCcFGwjcYjAzsxK3GMzMrMSJwczMSpwYzMysxInBzMxKnBjMzKzkfwEmtpXOY7yt5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# histogram of predicted probabilities\n",
    "\n",
    "# 8 bins\n",
    "plt.hist(y_pred_prob, bins=8)\n",
    "\n",
    "# x-axis limit from 0 to 1\n",
    "plt.xlim(0,1)\n",
    "plt.title('Histogram of predicted probabilities')\n",
    "plt.xlabel('Predicted probability of diabetes')\n",
    "plt.ylabel('Frequency')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We can see from the third bar\n",
    " - About 45% of observations have probability from 0.2 to 0.3\n",
    " - Small number of observations with probability > 0.5\n",
    " - This is below the threshold of 0.5\n",
    " - Most would be predicted \"no diabetes\" in this case\n",
    "- Solution\n",
    " - **Decrease the threshold** for predicting diabetes\n",
    "   - **Increase the sensitivity** of the classifier\n",
    "     - This would increase the number of TP\n",
    "       - More sensitive to positive instances\n",
    "       - Example of metal detector\n",
    "         - Threshold set to set off alarm for large object but not tiny objects\n",
    "         - YES: metal, NO: no metal\n",
    "         - We lower the threshold amount of metal to set it off\n",
    "         - It is now more sensitive to metal\n",
    "         - It will then predict YES more often"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.36752429, 0.28356344, 0.28895886, 0.4141062 , 0.15896027,\n",
       "       0.17065156, 0.49889026, 0.51341541, 0.27678612, 0.67189438])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_prob[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.36752429, 0.28356344, 0.28895886, 0.4141062 , 0.15896027,\n",
       "        0.17065156, 0.49889026, 0.51341541, 0.27678612, 0.67189438,\n",
       "        0.35755557, 0.74087965, 0.36050235, 0.23012363, 0.42654231,\n",
       "        0.19103515, 0.45763601, 0.1190141 , 0.43928953, 0.36961151,\n",
       "        0.44187989, 0.37611662, 0.19816022, 0.41677304, 0.15548281,\n",
       "        0.2531671 , 0.09743077, 0.69633712, 0.15358309, 0.2197836 ,\n",
       "        0.43094832, 0.34216058, 0.22396114, 0.38073543, 0.13342134,\n",
       "        0.38790216, 0.47049703, 0.16204743, 0.29548176, 0.30918161,\n",
       "        0.27299705, 0.38816583, 0.27353443, 0.28881041, 0.63471914,\n",
       "        0.02365251, 0.15820648, 0.23018375, 0.3484593 , 0.27580041,\n",
       "        0.33264104, 0.24880596, 0.74489512, 0.39001464, 0.41625545,\n",
       "        0.13575687, 0.18895376, 0.64777682, 0.18922131, 0.05685904,\n",
       "        0.63991547, 0.46636382, 0.1250972 , 0.26957602, 0.24919104,\n",
       "        0.30570396, 0.46376224, 0.20963095, 0.42847829, 0.40762264,\n",
       "        0.20169096, 0.27027066, 0.26255856, 0.57238263, 0.45467041,\n",
       "        0.27716152, 0.58001281, 0.41599488, 0.27276101, 0.34099223,\n",
       "        0.54626578, 0.37930723, 0.2992205 , 0.10059169, 0.32872602,\n",
       "        0.45101363, 0.16036979, 0.4896975 , 0.63230508, 0.40738404,\n",
       "        0.19794397, 0.19698021, 0.24463208, 0.11147185, 0.4158597 ,\n",
       "        0.21561856, 0.54124529, 0.48803602, 0.64652767, 0.33940658,\n",
       "        0.54263427, 0.16213824, 0.3778741 , 0.11311287, 0.34781987,\n",
       "        0.34042784, 0.1790985 , 0.21324812, 0.14710946, 0.23014102,\n",
       "        0.18404592, 0.52224649, 0.47099366, 0.28884248, 0.49325079,\n",
       "        0.41744473, 0.22915008, 0.27022911, 0.19243924, 0.7498713 ,\n",
       "        0.46500093, 0.6645454 , 0.42098599, 0.53564034, 0.16034702,\n",
       "        0.1435686 , 0.38142426, 0.33827314, 0.3630065 , 0.12842531,\n",
       "        0.28333693, 0.04005558, 0.18481139, 0.66716947, 0.46352874,\n",
       "        0.48715682, 0.19910794, 0.45861651, 0.23216721, 0.18369267,\n",
       "        0.26391994, 0.37492969, 0.12916506, 0.41413913, 0.42460858,\n",
       "        0.13832191, 0.20781694, 0.29477699, 0.15825099, 0.36016234,\n",
       "        0.23741449, 0.43350689, 0.20619881, 0.23162338, 0.61111541,\n",
       "        0.19731009, 0.80071498, 0.17808491, 0.36488735, 0.78618643,\n",
       "        0.44080614, 0.36559654, 0.11760138, 0.22843325, 0.47865069,\n",
       "        0.21320525, 0.51498521, 0.16122494, 0.23740119, 0.29374391,\n",
       "        0.16670048, 0.48716526, 0.29969894, 0.44651043, 0.50169902,\n",
       "        0.29246506, 0.61736228, 0.41593995, 0.25820945, 0.1741968 ,\n",
       "        0.33519541, 0.69606825, 0.32454368, 0.35730426, 0.2336947 ,\n",
       "        0.23738524, 0.38409318, 0.24691412, 0.27954552, 0.18501174,\n",
       "        0.2622362 , 0.27856926]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_prob.reshape(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict diabetes if the predicted probability is greater than 0.3\n",
    "from sklearn.preprocessing import binarize\n",
    "# it will return 1 for all values above 0.3 and 0 otherwise\n",
    "# results are 2D so we slice out the first column\n",
    "y_pred_class = binarize(y_pred_prob.reshape(1, -1), 0.3)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0., 0., 1., 0., 0., 1., 1., 0., 1.])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_class[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[118  12]\n",
      " [ 47  15]]\n"
     ]
    }
   ],
   "source": [
    "# previous confusion matrix (default threshold of 0.5)\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[80 50]\n",
      " [16 46]]\n"
     ]
    }
   ],
   "source": [
    "# new confusion matrix (threshold of 0.3)\n",
    "print(metrics.confusion_matrix(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The row totals are the same\n",
    "- The rows represent actual response values\n",
    " - 130 values top row\n",
    " - 62 values bottom row\n",
    "- Observations from the left column moving to the right column because we will have more TP and FP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7419354838709677\n"
     ]
    }
   ],
   "source": [
    "# sensitivity has increased (used to be 0.24)\n",
    "print (46 / float(46 + 16))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6153846153846154\n"
     ]
    }
   ],
   "source": [
    "# specificity has decreased (used to be 0.91)\n",
    "print(80 / float(80 + 50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Conclusion:**\n",
    "\n",
    "- **Threshold of 0.5** is used by default (for binary problems) to convert predicted probabilities into class predictions\n",
    "- Threshold can be **adjusted** to increase sensitivity or specificity\n",
    "- Sensitivity and specificity have an **inverse relationship**\n",
    " - Increasing one would always decrease the other\n",
    "- Adjusting the threshold should be one of the last step you do in the model-building process\n",
    " - The most important steps are\n",
    "    - Building the models\n",
    "    - Selecting the best model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Receiver Operating Characteristic (ROC) Curves"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Question:** Wouldn't it be nice if we could see how sensitivity and specificity are affected by various thresholds, without actually changing the threshold?\n",
    "\n",
    "**Answer:** Plot the ROC curve.\n",
    "\n",
    "- Receiver Operating Characteristic (ROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEXCAYAAACzhgONAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu8pXPd//HXewbp4Jgp0zA5TyblLFQadKC66aCiKKUm7qiQO8VPbt3uu9xUROGWSORQyRRSZKc04xBDZjQZTAzKcTCMw5jP74/vd7evWbPWta+97Wuttfd+Px+P9djrOn+u71p7fa/v4fpeigjMzMxaGdPpAMzMrLs5ozAzs1LOKMzMrJQzCjMzK+WMwszMSjmjMDOzUs4obFiTtLKkiyU9ISkkrdOGY+4raXGr6Yr7OEvSlUMfXefk9N+7jcebJ+nIwvRyks6U9EiOZcpITOdOcEbRQflLHPn1gqT5kn4kaUKTdV8t6bv5n+M5SQ9J+qmkzZqsu5ykgyRdL+lJSY9LulnSEZJWa8/Ztc0BwHbAm4HxwL0diOECYJnPrB0kXSnprE4cuwtsDXy7MP1B4KPAv5G+C38CvgB8qP2hjSzOKDrvD6Qv9UTSl3xz4KLiCpLWBm4Etif9MG4AvAd4HpghaZfCussDlwLHAhcCOwGbAkcA2wKfqPd0liZphZoPsSEwKyL+EhH/iIgXBrOTFxNnRCyKiH8OdnsbnIh4KCKeKszaELgvIv6UvwvPRcTjEfHYizlOG77D3S8i/OrQCzgLuLJh3kFAACsX5k0D/lGcV1h2WV720jx9KLAE2K7FMVcriWc54CjgTuBZ4D7gu4XlAezdsM2VwFmF6XnAfwHfAx4BbgDOBX7T5HiXA+cXpt8BXAssysf+IfDKknjn5Zh6Xz15/krAacBDwDOkTPadhe3Wyet/LKffU8DxLY4h4OvAg8BC4HzgYGBxYZ19G6ZXA34M3JPPZU7+XNT42QOH5HN9GvgZsEbD8fcEZubzmAd8C3h5YR/R8JqSl706L38IeDKn6w6F/S6f9zU/f9YPFD+LFmnxCuA7pFLbszmer7b6fpCu5mfmdPtHTrvxVWMAXg9cASzIn9HtwD4Nn/+R+X1PQzrMK/kfa5mmhX39IH/uDwAPdfq3otOvjgcwml+NX2LgNcDvgcWFH4PVgBd6/yGa7OOt+R9jtzw9s/EfYwDxnE36QdwHWJ9UAjm4sLxqRvEEcDSwETAZeFc+hwmF9V6dz3PXPL0T6cfyINKV4dbA1cA1FH5gG449jlTtcw2wJrB6nn9RjuNdwMbAicBzwOvy8nXyucwH9gbWA9ZtcYwv5B+pT+Tz+Y/8w1WWUawJfBnYAlg3H2Mh8MmGz/4J0kXAG4ApwB3AtIb9PpY/j/WAHYBbgXPy8lXyuV+Qj7kmsALwUmA2KePZilQCPYL0Y7xx3vaQfP5TSKXZrYEvlnw3RPoBvQt4XyGez7T6fuS0e3tOg+1IVUG/LywvjSGf63mk79B6wK7Aexu+a70ZxerA8cDdOR3GtfgfK03TvE4PKXM9NR/7DZ3+rej0q+MBjOZX/hIvzj8iT9N3NXR8YZ1t8rz3t9jH6nn5YXn6aeCkQcSyQd7PHiXrVM0ormpYZwzpqvnLhXmHkK7WxubpHuAbDdtNzMfcrJ80vLLJeby7Yb2bgDPz+3XyOv+vQrrMB45tmPdTSjKKFvs5EfhtQ9wLgVUK896Z49qwkJb7N+xnh7zOas3SvxDPfGC5hvm/A75TiOd3tMiEm8S/cz7uVgP5fjQs3zyvM6FKDMDjwL4l+5tH4QKKdHEyt5/vR5U07QH+BowZ6P/RSH25jaLzrgM2I2UIXwdmAP+vsFz9bB8N02oyr4ot8t/fDGLbRtcXJyJiCan6aZ/C7H2Ac6OvTWFr4IuSFva+SFfFkEoYVU3Of69pmH8NqSqjZZyNJK1MaqT+U8OiP/az3RhJh0uaKenhfC77A69tWHV2RDxemL42/91Y0ri8/rca0uTyvM4GJSFsTbqqXtCw7VvpS8sfkkoycyWdKumD/dTFbwk8FhE3lp17Ue51dIWkeyU9SV+69aZDfzEcD5whqUfS0ZK24EUYYJr+OX9vjVQnbZ21KCLm5ve3SdoIOAX4VJ53B6nNYRPg4ibbb5L/zin8bfxBHCrBshnX8k3We6rJvLOBwyRtSaoC2YylG9bHAN8Ezmmy7T8GHuoymmWgzeJs3IYm2/XnUOArpFLTTaRqjINJHRCq6r2I+wKpCq7R/H62vR14f5NlTwNExExJ65LahXYkXd1/XdK2EfFEi/1WTgdJE0ntP+cAxwAPA2uRSkArVIkhIr4u6VxgF1LV5FclHRcRRy57xEoGkqb9fTdGFZcous/RwCckbQUQEY+Srng+l69wG30V+Cfw2zz9Y2AnSds123lJ99ib8t93lsT2IKkdpXdfL6HvCr5URMzKx/h4fs2MiFsLq9wIvD4i5jZ5LaxyjGxW/rtDw/y3FpZVkq/27yN1vS1qnG60A/DriPhBRNycLwSalYo2bvhMt89/b4/Ui+peYFKLNHkmr/scMLZhvzeS6t+faLLd/YXzWxgRF0fE50ltGRsDb2txTn8GVu/9XlawNamt5IsRcW1EzCG1Sy2lvxgi4q6I+F5E7EHqaHFAxeMvYwBpag1cougyEfFXSb8C/od0pQXwOVL1x+/yDUazSFULB5OuxN4XEYvyuieSGnGvkHQMqb71IdI/4P6kK6kTmxx3br56+56kFYHppPaP7SOid/0rgf0lXUO6Sj6CfHVY0dnAkaRuvcc3LDsK+I2kb+f1niT9uH4IOLBwfqUi4k5JF+Xz+Czwd9KPyyak7scDdQLpKvevpGrB3UgNtGXmAPtI2pGU0XwceBOpEXWpcIEf5c90dVJJ8tKIuCMvPwL4gaQFwC9I6bYxqQPAZ/M6dwM7SlqfVKf/OKma72DgUklHkOrbX026Kr89In4h6TDgflLnh6eBvUgdDv7W4px+R+rKfYGkQ0gNwK8hNY6f0WT9O/L5HZq/V5uSPuN/KYtB0itIJcyf5XNclVSymM2LUyVNrVGnG0lG84smXffy/DeT/sl2LswbT/oh+TvpKvJh0j/R5k22X45UvL6RVIR+AriZVPpYtSSe5UntJPPyMeaTGz/z8jWBX+b93Uv6AW7WmN2qh9Yaeb/PA69usvyteX9P0tcd8js0NMr2l4bAyvR1j32W1t1j31LhMxoD/HdO76dIDdn9dY9dhXQPyxOkLsKn9KZrY9zAl0iN+otIVYvjGo7/PlKm/XTe30zgqMLy9UjtLwtZunvsK4HvkzKq5/Lfi3u/L8BnSaWEJ/K2NwC795MWKwHfzfE+R/oBP7ywvLHX0+fy92QRqX1il4YYW8YArEjq8XQ3qRvrg6TeXWu3+q5RoTG7Ypr2AGd0+vehm17KCWNmZtaU2yjMzKxUbRlFHpzrQUm3tVguSSdJmivp1hfb9c3MzOpRZ4niLFKdZCu7khorNwSmkupTzcysy9SWUUTENcCjJavsDvwokhnAqpLG1xWPmZkNTie7x05g6SGh5+d5DzSuKGkqqdTBiiuuuOXEiRPbEmC3W7JkCWPGuJkJnBZFTos+ozEt/vHUEp57AVZouLvmifvmPhwR4wazz05mFM2GpmjaBSsiTgdOB5g0aVLMmTOn2WqjTk9PD1OmTOl0GF3BadHHadFnNKbFR06bDsAFn136nltJfx/sPjuZ1c4H1i5Mr0W6+cbMzLpIJ0sU04ADJZ1Pumv18YhYptrJzKxO5113D5fMvK/TYQyZ2Q88weTxzUb7GbzaMgpJPyGNM7+GpPnA18gDyEXEqaQBw94NzCXdIfnJumIxM2vlkpn31fLj2imTx6/M7psN7ZN5a8soImKvfpYH6RZ/M7OOmjx+5WXq9K2PBwU0s46pu9pnwYJFfH/O9NJ1RlJpoi6jq9+YmXWV3mqfTqqjqmakcYnCzDqqzmqf1D3WVUovlksUZmZWyiUKMwM6003U7QPDg0sUZgZ0pr3A7QPDg0sUZvYv7iZqzTijMBvmGquMqnQJbcbVQNaKq57MhrmhqjJyNZC14hKF2QhQrDJyl1Abai5RmJlZKWcUZmZWyhmFmZmVckZhZmal3JhtNkz1dot1t1arm0sUZsNUMZNwt1ark0sUZsOY76S2dnCJwszMSjmjMDOzUs4ozMyslDMKMzMr5cZssy5T9QFC7hZr7eIShVmXqToarLvFWru4RGHWhdzt1bqJSxRmZlbKJQqzLlBsl3Dbg3UblyjMukCxXcJtD9ZtXKIw6xJul7Bu5YzCbIhV7d5a5Oom62alVU+StpZ0oqSbJD0g6S5J0yR9VtJK7QrSbDip2r21yNVN1s1aligk/Qp4BLgEOAF4EFgR2AjYEbhU0nER8at2BGo2nLgayUaSsqqn/SLinw3zngGuz69vSnpVbZGZmVlXaJlR9GYSkvYHfhIRjzdZ58EaYzPrOLc3mFXrHrsOcJOk8yS9veZ4zLqK2xvMKvR6iojDJX0V2BXYX9L3gZ8AZ0bEvJrjM+s4tzfYaFepe2xELJE0D5gHvAEYD1wi6bKI+Eqr7STtApwIjAXOiIhvNCyfCJwNrJrXOTwiLhvEeZgNSn9VS65GMqtQ9STp3yVdT/rB/zPwxoj4DLA58JGS7cYCp5BKIpOBvSRNbljtSODCiNgc2BP43qDOwmyQ+qtacjWSWbUSxVrAnhFxV3FmLmXsVrLdNsDc3u0knQ/sDswu7gbovVxbBbi/auBmQ8VVS2blqmQUr2nMJCSdFRH7RsRtJdtNAO4tTM8H3tSwztHAbyQdBLwcaNpYLmkqMBVg3Lhx9PT0VAh75Fu4cKHTIhtsWixYsAhgRKWjvxd9nBZDo0pG8cbihKQxwNYVtlOTedEwvRdwVkScIGk74BxJm0TEkqU2ijgdOB1g0qRJMWXKlAqHH/l6enpwWiRV0qJZe8T9i55l8viVmTJl5JQo/L3o47QYGi3bKCR9WdJjwBslPZpfjwEPA1UanOcDaxem12LZqqX9gAsBImI66c7vNQYQv1llzdoj3AZh1r+yEsVxpKE7/gc4vHdmRLxQcd83ABtKWhe4j9RY/dGGde4BdgbOkrQxKaN4qOL+zQbM7RFmA1eWUWwQEXdIOgd4fe9MKdUoRcStZTuOiMWSDgSuIHV9PTMiZkk6BrgxIqYBhwL/J+lgUrXUvhHRWD1lZmYdVJZRHE6qGjqlybIAduhv5/meiMsa5h1VeD8beHOlSM3MrCPKxnraL/99a/vCMTOzblPlhrubJB0m6bXtCMjMzLpLle6xHyLdgT1N0tPABcBFETGwITXNhkCrITcWLFjE9+dML93Ww3GYDU6/JYqIuDMi/jsiNgU+BWwJ/L32yMyaGMxorr3cFdZscCoNCihpLeDDpJLFcsARdQZlVqZZF9d0Y5W7vZrVod+MQtK1wErARcA+EfG32qMyM7OuUaVE8dl+xnQyM7MRrGVGIWmviPgJsJOknRqXR8RJtUZmZmZdoaxEsVr+O67JMt89bWY2SpTdcNf7EKFLI2JGcZmkbWuNykad/p4018tdXM3ar9/usTR/6lyzYT3MBq1qt1d3cTVrv7I2im2A7YBxkj5fWLQysHzdgdno45FdzbpTWRvFy0nPhliOpdspniTdrW1mZqNAWRvF1cDVkn7Y+ChUMzMbPcqqnk6IiEOBEyQt08spIj5Qa2RmZtYVyqqeLsh/T25HIGZm1p3Kqp6uz3+v6p0naRVgQn7gkJmZjQJVnkdxlaSVJa0G/AU4T9L/1h+amZl1gyr3UaweEU8AHwDOjojNgHfVG5aZmXWLKhnFcpLGkbrE/rLmeMzMrMtUySiOBX4P3BMR10taD7i73rDMzKxb9DvMeEScD5xfmL4L2L3OoMzMrHtUeXDRGqRHoK5TXD8iptYXlpmZdYsqDy66BJgB/BF4od5wbKTqb3RYjwpr1r2qZBQvz3domw1a7+iwrTIDjwpr1r2qZBSXS3pnRPym9mhsRPPosGbDU5VeT/sDv5a0UNKjkh6T9GjdgZmZWXeoUqJYo/YozMysa/VbooiIF0g32305vx8PbFZ3YGZm1h2qjPV0MrAjsE+e9TRwap1BmZlZ96hS9bR9RGwh6WaAiHhU0go1x2VmZl2iSmP285LGAAEg6ZXAklqjMjOzrlElozgF+BkwTtJ/km68+2atUZmZWdeoMtbTjyT9GXh7nvWhiLit3rDMzKxbtCxRSFpR0liAiJgFXEqqclqv6s4l7SJpjqS5kg5vsc6HJc2WNEvSeQOM38zMalZW9XQFsD6ApPWB64HJwCGSju1vxzmTOQXYNW+3l6TJDetsCHwFeHNEvB744mBOwszM6lOWUaweEX/L7z8BnB8RB5CebrdbhX1vA8yNiLsi4jnSUOWNw5N/BjglIh4DiIgHBxS9mZnVrqyNIgrvdwJOAIiIZyVV6fU0Abi3MD0feFPDOhsBSLoWGAscHRG/btyRpKnAVIBx48bR09NT4fAj38KFC7syLXrufZ7p9y9eat49Ty5h4kpjaou3W9OiE5wWfZwWQ6Mso5gl6RvAfaQf9N8ASFoFUIV9N1snGqaXAzYEpgBrAX+QtElELFhqo4jTgdMBJk2aFFOmTKlw+JGvp6eHbkyL7582nfsXLT1S7Kqrwu6bTWDKmybWcsxuTYtOcFr0cVoMjbKM4tPAwcDrgF0i4qk8fxPgWxX2PR9YuzC9FnB/k3VmRMTzwN2S5pAyjhsq7N+6mEeKNRs5WmYUOWP4rybzrwWurbDvG4ANJa1LKpXsCXy0YZ1fAHsBZ+Un6W0E3FUtdDMza4ey7rG/kLSrpGUyE0mvlXSUpE+12j4iFgMHknpP3Q5cGBGzJB0jqbcx/ArgEUmzgauBwyLikRdzQmZmNrTKqp4+BxwKnCLpn8BDwIqk+yjuIfVW+lnZziPiMuCyhnlHFd4HcEh+mZlZFyqrerqP/CMuaQPS8OKLgDkR8WSb4jMzsw6rMnosETEXmFtzLGZm1oWqDApoZmajmDMKMzMrVSmjkLRCbqcwM7NRpt82CknvId1gtwKwrqTNgK9FxPvrDs6633nX3cMlM+9bat7sB5a+K9vMhrcqJYpjSGM0LQCIiJmASxcGwCUz72P2A08sNW/y+JXZfbMJHYrIzIZalV5Pz0fEAmmpoZsax2yyUczDdZiNbFUyitslfRgYk4fj+AIwo96wrJOaVSe14moms5GvStXTgcCWpKfb/Rx4hpRZ2AjVrDqpFVczmY18VUoU74qILwNf7p0h6QOkTMNGKFcnmVmvKiWKI5vMO2KoAzEzs+7UskQh6V3ALsAEScXnT6xMqoYyM7NRoKzq6UHgNlKbxKzC/CeBw+sMyszMukfZ6LE3AzdLOjcinmljTGZm1kWqNGZPkHQsMJn0PAoAImKj2qKyWlTt9uour2ZWVKUx+yzgh4CAXYELgfNrjMlqUrXbq7u8mllRlRLFyyLiCknHR8SdwJGS/lB3YFYPd3s1s4GqklE8qzR+x52S9gfuA15Vb1hmZtYtqmQUBwOvAD4PHAusAnyqzqCsucY2hgULFvH9OdMrb++2BzMbjH4zioi4Lr99EtgHQNJadQZlzfW2MQz2x95tD2Y2GKUZhaStgQnAHyPiYUmvJw3lsRPgzKIDim0MPT09TJni9gYzq1fLXk+S/gc4F/gY8GtJRwBXA7cA7hrbRudddw8fOW165YH6zMyGUlmJYndg04hYJGl14P48Pac9oVmvYpWTq47MrN3KMopnImIRQEQ8KumvziQ6x91azaxTyjKK9ST1DiUuYJ3CNBHxgVojMzOzrlCWUXywYfrkOgMZzfobWsPdWs2sk8oGBbyqnYGMZv11e3XbhJl1UpUb7qwN3AZhZt3KGUUH9VY5uWrJzLpZldFjAZD0kjoDGY3c7dXMhoN+SxSStgF+QBrjaaKkTYFPR8RBdQc3GrjKycy6XZUSxUnAe4FHACLiFmDHOoMyM7PuUSWjGBMRf2+Y90IdwZiZWfep0ph9b65+CkljgYOAv9UblpmZdYsqJYoDgEOAicA/gW3zvH5J2kXSHElzJR1est4ekkLSVlX2a2Zm7VOlRLE4IvYc6I5z6eMU4B3AfOAGSdMiYnbDeiuRHop03bJ7MTOzTqtSorhB0mWSPpF/1KvaBpgbEXdFxHPA+aQRaRt9HTgOeGYA+zYzszap8oS79SVtD+wJ/KekmcD5EXF+P5tOAO4tTM8H3lRcQdLmwNoR8StJX2q1I0lTgakA48aNo6enp7+wh4UFCxYBDPp8Fi5cOGLS4sVyWvRxWvRxWgyNSndmR8SfgD9JOhr4DumBRv1lFGq2q38tlMYA3wb2rXD804HTASZNmhRTpkypEnbX633e9WCfUpeecDdlCCMavpwWfZwWfZwWQ6PKDXevIFUZ7QlsDFwCbF9h3/OBtQvTa5EeftRrJWAToEcSwJrANEm7RcSNlaIfhoojxXroDjMbDqqUKG4DfgkcFxF/GMC+bwA2lLQucB8po/lo78KIeBxYo3daUg/wpZGcScDSw3Z46A4zGw6qZBTrRcSSge44IhZLOhC4AhgLnBkRsyQdA9wYEdMGus+RwsN2mNlw0jKjkHRCRBwK/ExSNC6v8oS7iLgMuKxh3lEt1p3Sb7RmZtZ2ZSWKC/JfP9nOzGwUK3vC3fX57cYRsVRmkauU/AQ8M7NRoMoNd59qMm+/oQ7EzMy6U1kbxUdIPZXWlfTzwqKVgAV1B2ZmZt2hrI3ietIzKNYijdnU60ng5jqDMjOz7lHWRnE3cDdwZfvCMTOzblNW9fT7iHibpMcoDL1BGpojImL12qMbIXw3tpkNZ2VVT72PO12jZB2rwHdjm9lwVlb11Hs39trA/RHxnKS3AG8Efgw80Yb4RgzfjW1mw1WV7rG/ID0GdX3gR6SBAc+rNSozM+saVcZ6WhIRz0v6APCdiDhJkns9tVBsj+jldgkzG86qlCgWS/oQsA/wqzxv+fpCGt562yOK3C5hZsNZlRLFp4B/Jw0zflceNvwn9YY1vLk9wsxGkiqPQr1N0ueBDSS9jvQc7GPrD83MzLpBlSfcvRU4h/TwIQFrStonIq6tOzgzM+u8KlVP3wbeHRGzASRtTMo4tqozMDMz6w5VGrNX6M0kACLidmCF+kIyM7NuUqVEcZOk00ilCICP4UEBl9HbLdZdYc1spKmSUewPfB74D1IbxTXAd+sMajgqZhLuCmtmI0lpRiHpDcD6wMURcVx7Qhq+3C3WzEaistFjv0p6kt1NwNaSjomIM9sWWZdpdsd1kauczGykKmvM/hjwxoj4ELA1cEB7QupOze64LnKVk5mNVGVVT89GxFMAEfGQpCo9pEY0Vy2Z2WhUllGsV3hWtoD1i8/OjogP1BqZmZl1hbKM4oMN0yfXGYiZmXWnsgcXXdXOQMzMrDuN+nYHMzMrV+WGu1Gr2CXW3V/NbLSqXKKQ9JI6A+lGxS6x7v5qZqNVlWHGtwF+AKwCTJS0KfDpiDio7uC6gbvEmtloV6VEcRLwXuARgIi4BdixzqDMzKx7VGmjGBMRf5dUnPdCTfF0BY8Ea2bWp0pGcW+ufgpJY4GDgL/VG1ZneSRYM7M+VTKKA0jVTxOBfwJXMgrGfXLbhJlZ0m9GEREPAnsOZueSdgFOBMYCZ0TENxqWHwJ8GlgMPAR8KiL+PphjvVjuCmtm1lyVXk//B0Tj/IiY2s92Y4FTgHcA84EbJE0rPlaV9KS8rSLiaUkHAMcBHxlA/EOmWN3kKiczsz5Vqp6uLLxfEXg/cG+F7bYB5kbEXQCSzgd2B4rP3766sP4MYO8K+62Nq5vMzJZVperpguK0pHOA31bY9wSWzlDmA28qWX8/4PJmCyRNBaYCjBs3jp6engqHH5gFCxYB1LLvuixcuHBYxVsnp0Ufp0Ufp8XQGMwQHusCr62wnprMW6YKC0DS3sBWwNuaLY+I04HTASZNmhRTpkypFCj0/2S6XvcvepbJ41dmypThU6Lo6elhIGkxkjkt+jgt+jgthkaVNorH6PuBHwM8ChxeYd/zgbUL02sB9zfZ/9uBI4C3RcSzFfY7IFXvh3C7hJlZc6UZhdJddpsCvZfkSyKiaamgiRuADSWtm7ffE/how/43B04Ddsm9q2rhtgczs8ErzSgiIiRdHBFbDnTHEbFY0oHAFaTusWdGxCxJxwA3RsQ04H+BVwAX5Tu/74mI3QZ8Fg3c1dXMbOhUaaO4XtIWEXHTQHceEZcBlzXMO6rw/u0D3WcV7upqZjZ0WmYUkpaLiMXAW4DPSLoTeIrUSB0RsUWbYhwUVzeZmQ2NshLF9cAWwPvaFIuZmXWhsoxCABFxZ5tiMTOzLlSWUYzLYzE1FRHfqiEeMzPrMmUZxVhSj6RmN86ZmdkoUZZRPBARx7QtEjMz60plj0J1ScLMzEozip3bFoWZmXWtlhlFRDzazkDMzKw7lZUozMzMnFGYmVk5ZxRmZlbKGYWZmZVyRmFmZqWcUZiZWSlnFGZmVqrKg4uGjd4n2/mpdmZmQ2dElSiKmYSfamdmNjRGVIkC/GQ7M7OhNuwzit7qJsBVTmZmNRj2VU+91U2Aq5zMzGow7EsU4OomM7M6DfsShZmZ1WvYlijcFdbMrD2GbYnCXWHNzNpj2JYowG0TZmbtMGxLFGZm1h7OKMzMrJQzCjMzK+WMwszMSjmjMDOzUs4ozMyslDMKMzMr5YzCzMxKOaMwM7NStWYUknaRNEfSXEmHN1n+EkkX5OXXSVqnznjMzGzgassoJI0FTgF2BSYDe0ma3LDafsBjEbEB8G3gm3XFY2Zmg1NniWIbYG5E3BURzwHnA7s3rLM7cHZ+/1NgZ0kq2+k/nlrCR06b/q+HFZmZWb3qHBRwAnBvYXo+8KZW60TEYkmPA68EHi6uJGkqMDVPPnvh/tvfBnAbcOH+Qx/4MLIGDWk1ijkt+jgt+jgt+kwa7IZ1ZhTNSgYxiHWIiNOB0wEk3RgRW7348IY/p0Ufp0Ufp0Ufp0UfSTcOdts6q57mA2sXptcC7m+1jqTlgFWAR2uMyczMBqjOjOIGYENJ60paAdgTmNawzjTgE/n9HsB/ZGOjAAAJU0lEQVTvImKZEoWZmXVObVVPuc3hQOAKYCxwZkTMknQMcGNETAN+AJwjaS6pJLFnhV2fXlfMw5DToo/Too/Too/Tos+g00K+gDczszK+M9vMzEo5ozAzs1Jdm1F4+I8+FdLiEEmzJd0q6SpJr+1EnO3QX1oU1ttDUkgasV0jq6SFpA/n78YsSee1O8Z2qfA/MlHS1ZJuzv8n7+5EnHWTdKakByXd1mK5JJ2U0+lWSVtU2nFEdN2L1Ph9J7AesAJwCzC5YZ1/B07N7/cELuh03B1Mix2Bl+X3B4zmtMjrrQRcA8wAtup03B38XmwI3Ayslqdf1em4O5gWpwMH5PeTgXmdjrumtNgB2AK4rcXydwOXk+5h2xa4rsp+u7VEUcvwH8NUv2kREVdHxNN5cgbpnpWRqMr3AuDrwHHAM+0Mrs2qpMVngFMi4jGAiHiwzTG2S5W0CGDl/H4Vlr2na0SIiGsovxdtd+BHkcwAVpU0vr/9dmtG0Wz4jwmt1omIxUDv8B8jTZW0KNqPdMUwEvWbFpI2B9aOiF+1M7AOqPK92AjYSNK1kmZI2qVt0bVXlbQ4Gthb0nzgMuCg9oTWdQb6ewLUO4THizFkw3+MAJXPU9LewFbA22qNqHNK00LSGNIoxPu2K6AOqvK9WI5U/TSFVMr8g6RNImJBzbG1W5W02As4KyJOkLQd6f6tTSJiSf3hdZVB/W52a4nCw3/0qZIWSHo7cASwW0Q826bY2q2/tFgJ2ATokTSPVAc7bYQ2aFf9H7kkIp6PiLuBOaSMY6Spkhb7ARcCRMR0YEXSgIGjTaXfk0bdmlF4+I8+/aZFrm45jZRJjNR6aOgnLSLi8YhYIyLWiYh1SO01u0XEoAdD62JV/kd+QerogKQ1SFVRd7U1yvaokhb3ADsDSNqYlFE81NYou8M04OO599O2wOMR8UB/G3Vl1VPUN/zHsFMxLf4XeAVwUW7PvycidutY0DWpmBajQsW0uAJ4p6TZwAvAYRHxSOeirkfFtDgU+D9JB5OqWvYdiReWkn5CqmpcI7fHfA1YHiAiTiW1z7wbmAs8DXyy0n5HYFqZmdkQ6taqJzMz6xLOKMzMrJQzCjMzK+WMwszMSjmjMDOzUs4oRilJL0iaWXitU7LuOq1GoxzgMXvyCJ+35GElJg1iH/tL+nh+v6+k1xSWnSFp8hDHeYOkzSps80VJLxvEsb4jaYf8/sA8qmfk+x4Guq9JOfaZkm6XNKRPd5O0W+/IrJLGKY3afLOkt0q6TNKqJdu2/NxKtrlS0mpDdwY2aJ0e7dCvzryAhQNYdx1ajEY5wGP2kEdzBaYC04Zqf0OcNsU4Pwn8tsI284A1Bnic1YEZhenNc1oPeF95+yuA3QvTb6jx+7MncHadnxvphtoj6joHv6q/XKKwf8klhz9Iuim/tm+yzuslXZ+vWm+VtGGev3dh/mmSxvZzuGuADfK2O+cr078ojaf/kjz/G+p7zsbxed7Rkr4kaQ/SuFbn5mO+NF9NbyXpAEnHFWLeV9J3BxnndAqDpkn6vqQblZ7v8J953ueB1wBXS7o6z3unpOk5HS+S9Iom+94D+HXvRETcHBHz+omnzHjSEA29+/tLjmVfSZdI+nUuKX2tcD5N00Pp+Q435VLVVYX9nJxLWMcB7y6k/bzeUpCkj+fP7BZJ5+R5rT6390i6uBDPOyT9PE9OI43RZJ3W6ZzKr868SHfqzsyvi/O8lwEr5vcbku5qhUKJAvgu8LH8fgXgpcDGwC+B5fP87wEfb3LMHvqu1A8DLiANpXAvsFGe/yPgi6Sr7Tn03RS6av57NPClxv0Vp4FxpGGne+dfDrxlkHF+EfjvwrLV89+xeb035ul55FIAaQyha4CX5+kvA0c1Oc7ZwL81mf+vfQ3wM/0kaRTly4GDC2m2L/AAaXTllwK35XRqmh45/e4F1m04532BkxvfF2MGXp8/tzUatm36uZEGqfsrMC5Pn1dME+AO4JWd/n8Z7a+uHMLD2mJRRDTWvS8P9F4xvkAaG6jRdOAISWsBP4+IOyTtDGwJ3KA0hMhLgVZjTp0raRHph+UgYBJwd0T8LS8/G/gccDLpeRJnSLoUqDxseEQ8JOkupbFs7sjHuDbvdyBxvpyUIRSfAvZhSVNJw9+MJz0E59aGbbfN86/Nx1mBlG6NxjOE4w1FxA8lXQHsQnruwGclbZoX/zby8B35iv0twGKap8e2wDWRBhIkIgYy2OZOwE8j4uEq20ZE5FLH3pJ+CGxHyqx6PUgqrY24oUeGE2cUVnQw8E9gU1JHh2Ue/BMR50m6DngPcIWkT5OuCs+OiK9UOMbHojBIn6SmzxCJNH7PNqSB3PYEDiT9CFV1AfBh0tXqxfkHaUBxkp6U9g3gFOADktYFvgRsHRGPSTqLVCJqJNIPc3/VJotabN9S/jHdHLg/IpZ5nGdE3A+cCZyp1AFhk95FjavS4nOTtFuT9SuHOIhtf0gq2TwDXBTp+TK9ViSlk3WQ2yisaBXggUhj9O9DuppeiqT1gLsi4iRSHfIbgauAPSS9Kq+zuqo/t/uvwDqSNsjT+wC/z3X6q0TEZaTqn2Y9j54kDS3ezM+B95HquC/I8wYUZ0Q8DxwJbKs04ujKwFPA45JeDezaIpYZwJt7z0nSyyQ1K53dTm6nqSoiPhkRmzXLJHK7wvL5/Zqkqqb78uJ35PN9KSldrqV1ekwH3pYzRiStPoAQryKVul5Zsu1Sn1vO3O4npfVZhfMRsCap9Gkd5IzCir4HfELSDFK101NN1vkIcJukmcDrSI9VnE36J/+NpFuB35KqVfoVEc+Q6tYvkvQXYAlwKumH5Fd5f78nlXYanQWc2tug2rDfx4DZwGsj4vo8b8BxRsQi4ARS/fotpGdQzyJdtV9bWPV04HJJV0fEQ6Q6/J/k48wgpVWjS0kjfQKpUVxpxM+1gFslnVEWWxPvJH02t5B6QB0WEf/Iy/4InENqk/pZRNzYKj1y/FOBn+d9XdB4oFYiYhZwLCmzvwX4VpPVzmLZz+1c4N4cU68tSb3CFjfuwNrLo8eadZCkPwLvjRqfOidpX1Lj8YF1HePFknQycHNE/KAw70RSF+qrOheZgUsUZp12KDCx00F0kqQ/k6owf9yw6DZnEt3BJQozMyvlEoWZmZVyRmFmZqWcUZiZWSlnFGZmVsoZhZmZlfr/s4Fo3fWvTTEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# IMPORTANT: first argument is true values, second argument is predicted probabilities\n",
    "\n",
    "# we pass y_test and y_pred_prob\n",
    "# we do not use y_pred_class, because it will give incorrect results without generating an error\n",
    "# roc_curve returns 3 objects fpr, tpr, thresholds\n",
    "# fpr: false positive rate\n",
    "# tpr: true positive rate\n",
    "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_prob)\n",
    "\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.rcParams['font.size'] = 12\n",
    "plt.title('ROC curve for diabetes classifier')\n",
    "plt.xlabel('False Positive Rate (1 - Specificity)')\n",
    "plt.ylabel('True Positive Rate (Sensitivity)')\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- ROC curve can help you to **choose a threshold** that balances sensitivity and specificity in a way that makes sense for your particular context\n",
    "- You can't actually **see the thresholds** used to generate the curve on the ROC curve its"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function that accepts a threshold and prints sensitivity and specificity\n",
    "def evaluate_threshold(threshold):\n",
    "    print('Sensitivity:', tpr[thresholds > threshold][-1])\n",
    "    print('Specificity:', 1 - fpr[thresholds > threshold][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.24193548387096775\n",
      "Specificity: 0.9076923076923077\n"
     ]
    }
   ],
   "source": [
    "evaluate_threshold(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.7258064516129032\n",
      "Specificity: 0.6153846153846154\n"
     ]
    }
   ],
   "source": [
    "evaluate_threshold(0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AUC\n",
    "\n",
    "AUC is the **percentage** of the ROC plot that is **underneath the curve**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7245657568238213\n"
     ]
    }
   ],
   "source": [
    "# IMPORTANT: first argument is true values, second argument is predicted probabilities\n",
    "print(metrics.roc_auc_score(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- AUC is useful as a **single number summary** of classifier performance\n",
    "- Higher value = better classifier\n",
    "- If you randomly chose one positive and one negative observation, AUC represents the likelihood that your classifier will assign a **higher predicted probability** to the positive observation\n",
    "- AUC is useful even when there is **high class imbalance** (unlike classification accuracy)\n",
    " - Fraud case\n",
    "    - Null accuracy almost 99%\n",
    "    - AUC is useful here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Paschal\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7378233618233618"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate cross-validated AUC\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use both of these whenever possible\n",
    "\n",
    "1. **Confusion matrix advantages:**\n",
    "\n",
    " - Allows you to calculate a **variety of metrics**\n",
    " - Useful for **multi-class problems** (more than two response classes)\n",
    "2. **ROC/AUC advantages:**\n",
    "\n",
    " - Does not require you to **set a classification threshold**\n",
    " - Still useful when there is **high class imbalance**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}