spam or ham.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://stackabuse.com/the-naive-bayes-algorithm-in-python-with-scikit-learn/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_table('SMSSpamCollection',  \n",
    "                   sep='\\t', \n",
    "                   header=None,\n",
    "                   names=['label', 'message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5567</th>\n",
       "      <td>spam</td>\n",
       "      <td>This is the 2nd time we have tried 2 contact u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5568</th>\n",
       "      <td>ham</td>\n",
       "      <td>Will ü b going to esplanade fr home?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5569</th>\n",
       "      <td>ham</td>\n",
       "      <td>Pity, * was in mood for that. So...any other s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5570</th>\n",
       "      <td>ham</td>\n",
       "      <td>The guy did some bitching but I acted like i'd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5571</th>\n",
       "      <td>ham</td>\n",
       "      <td>Rofl. Its true to its name</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5572 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                            message\n",
       "0      ham  Go until jurong point, crazy.. Available only ...\n",
       "1      ham                      Ok lar... Joking wif u oni...\n",
       "2     spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3      ham  U dun say so early hor... U c already then say...\n",
       "4      ham  Nah I don't think he goes to usf, he lives aro...\n",
       "...    ...                                                ...\n",
       "5567  spam  This is the 2nd time we have tried 2 contact u...\n",
       "5568   ham               Will ü b going to esplanade fr home?\n",
       "5569   ham  Pity, * was in mood for that. So...any other s...\n",
       "5570   ham  The guy did some bitching but I acted like i'd...\n",
       "5571   ham                         Rofl. Its true to its name\n",
       "\n",
       "[5572 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['label'] = df['label'].map({'ham':0, 'spam':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['message'] = df['message'].map(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>go until jurong point crazy available only in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>ok lar joking wif u oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>free entry in 2 a wkly comp to win fa cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>u dun say so early hor u c already then say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>nah i dont think he goes to usf he lives aroun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5567</th>\n",
       "      <td>1</td>\n",
       "      <td>this is the 2nd time we have tried 2 contact u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5568</th>\n",
       "      <td>0</td>\n",
       "      <td>will ü b going to esplanade fr home</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5569</th>\n",
       "      <td>0</td>\n",
       "      <td>pity  was in mood for that soany other suggest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5570</th>\n",
       "      <td>0</td>\n",
       "      <td>the guy did some bitching but i acted like id ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5571</th>\n",
       "      <td>0</td>\n",
       "      <td>rofl its true to its name</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5572 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                            message\n",
       "0         0  go until jurong point crazy available only in ...\n",
       "1         0                            ok lar joking wif u oni\n",
       "2         1  free entry in 2 a wkly comp to win fa cup fina...\n",
       "3         0        u dun say so early hor u c already then say\n",
       "4         0  nah i dont think he goes to usf he lives aroun...\n",
       "...     ...                                                ...\n",
       "5567      1  this is the 2nd time we have tried 2 contact u...\n",
       "5568      0                will ü b going to esplanade fr home\n",
       "5569      0  pity  was in mood for that soany other suggest...\n",
       "5570      0  the guy did some bitching but i acted like id ...\n",
       "5571      0                          rofl its true to its name\n",
       "\n",
       "[5572 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['message'] = df['message'].str.replace('[^\\w\\s]', \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['message'] = df['message'].apply(nltk.word_tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "df['message'] = df['message'].apply(lambda x: [stemmer.stem(y) for y in x] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[go, until, jurong, point, crazi, avail, onli,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>[ok, lar, joke, wif, u, oni]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>[free, entri, in, 2, a, wkli, comp, to, win, f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>[u, dun, say, so, earli, hor, u, c, alreadi, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>[nah, i, dont, think, he, goe, to, usf, he, li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5567</th>\n",
       "      <td>1</td>\n",
       "      <td>[thi, is, the, 2nd, time, we, have, tri, 2, co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5568</th>\n",
       "      <td>0</td>\n",
       "      <td>[will, ü, b, go, to, esplanad, fr, home]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5569</th>\n",
       "      <td>0</td>\n",
       "      <td>[piti, wa, in, mood, for, that, soani, other, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5570</th>\n",
       "      <td>0</td>\n",
       "      <td>[the, guy, did, some, bitch, but, i, act, like...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5571</th>\n",
       "      <td>0</td>\n",
       "      <td>[rofl, it, true, to, it, name]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5572 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                            message\n",
       "0         0  [go, until, jurong, point, crazi, avail, onli,...\n",
       "1         0                       [ok, lar, joke, wif, u, oni]\n",
       "2         1  [free, entri, in, 2, a, wkli, comp, to, win, f...\n",
       "3         0  [u, dun, say, so, earli, hor, u, c, alreadi, t...\n",
       "4         0  [nah, i, dont, think, he, goe, to, usf, he, li...\n",
       "...     ...                                                ...\n",
       "5567      1  [thi, is, the, 2nd, time, we, have, tri, 2, co...\n",
       "5568      0           [will, ü, b, go, to, esplanad, fr, home]\n",
       "5569      0  [piti, wa, in, mood, for, that, soani, other, ...\n",
       "5570      0  [the, guy, did, some, bitch, but, i, act, like...\n",
       "5571      0                     [rofl, it, true, to, it, name]\n",
       "\n",
       "[5572 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# This converts the list of words into space-separated strings\n",
    "df['message'] = df['message'].apply(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer()\n",
    "counts = count_vect.fit_transform(df['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "transformer = TfidfTransformer().fit(counts)\n",
    "\n",
    "counts = transformer.transform(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(counts, df['label'], test_size=0.1, random_state=69)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "model = MultinomialNB().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9480286738351255\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "predicted = model.predict(X_test)\n",
    "\n",
    "print(np.mean(predicted == y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[482   0]\n",
      " [ 29  47]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, plot_roc_curve\n",
    "\n",
    "print(confusion_matrix(y_test, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.roc_curve.RocCurveDisplay at 0x1948748>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgV1bX38e9iRkYZ9FUGGwQHEGhIC5oEonI1gIqCJoBGVKJIIuKYV0yikuE6BKMXryMaRO+LtFcjikTD9cYYUWQSARkFEbEZZNAgKDPr/aOqO4fu093V0HWa7vp9nuc8fapqV521u/s5q/auqr3N3RERkeSqVtEBiIhIxVIiEBFJOCUCEZGEUyIQEUk4JQIRkYSrUdEBlFWzZs08KyurosMQEalUPvjggy3u3jzdtkqXCLKyspg3b15FhyEiUqmY2WfFbVPXkIhIwikRiIgknBKBiEjCKRGIiCScEoGISMLFlgjMbIKZbTKzxcVsNzN72MxWmdkiM+sWVywiIlK8OFsEE4E+JWzvC7QPX8OBx2OMRUREihHbcwTu/o6ZZZVQ5CLgOQ/GwZ5lZo3N7Dh33xBXTCJRPD97La8uWFfRYYgU0eH4htx9YcdyP25FXiNoAXyespwXrivCzIab2Twzm7d58+aMBCfJ9eqCdSzd8HVFhyGSMRX5ZLGlWZd2lhx3Hw+MB8jJydFMOhK7Dsc15IXrzqzoMEQyoiITQR7QKmW5JbC+gmKRiJLQbbJ0w9d0OK5hRYchkjEV2TU0FRga3j10BrBN1weOfEnoNulwXEMuyk7bSylSJcXWIjCzycBZQDMzywPuBmoCuPsTwOtAP2AV8C1wdVyxHGkq81l1/tmyuk1Eqo447xoaUsp2B66P6/OPZPln1ZWx+0FnyyJVT6Ubhrqq0Fm1iBwplAgyKL9LqLK2BkSkatJYQxmUmgTUvSIiRwq1CDJMXUIicqRRIjgMZb37R11CInIkUtfQYSjrPfXqEhKRI5FaBCkO9QxfXT0iUpmpRZBCZ/gikkRqERSiM3wRSRolAnR/v4gkm7qG0P39IpJsahGE1CUkIkmlFoGISMIpEYiIJFziE8Hzs9cy+9MvKzoMEZEKk/hEkP8AmS4Si0hSJT4RAPRo04TLerSu6DBERCpEYu8a0rMDIiKBxLYI9OyAiEggsS0C0LMDIiKQ4BaBiIgElAhERBIucV1DukgsInKwxLUIdJFYRORgiWsRgC4Si4ikSlyLQEREDqZEICKScEoEIiIJp0QgIpJwSgQiIgmnRCAiknCxJgIz62NmK8xslZmNTrO9kZm9ZmYLzWyJmV0dZzwiIlJUbInAzKoDjwJ9gQ7AEDPrUKjY9cBSd+8CnAX80cxqxRHP87PXMujJ91m64es4Di8iUmnF2SLoDqxy99XuvgfIBS4qVMaBBmZmQH3gS2BfHMHoiWIRkfTifLK4BfB5ynIe0KNQmUeAqcB6oAEwyN0PFD6QmQ0HhgO0bn3oM4npiWIRkaLibBFYmnVeaPmHwALgeCAbeMTMiowE5+7j3T3H3XOaN29e/pGKiCRYnIkgD2iVstyS4Mw/1dXAyx5YBXwKnBJjTCIiUkiciWAu0N7M2oQXgAcTdAOlWgv0BjCzY4GTgdUxxiQiIoXEdo3A3feZ2UhgOlAdmODuS8xsRLj9CeB3wEQz+4igK+l2d98SV0wiIlJUrMNQu/vrwOuF1j2R8n49cF6cMYiISMn0ZLGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgkXOREYGb14gxEREQqRqmJwMy+a2ZLgWXhchczeyz2yEREJCOitAgeIphAZiuAuy8EesUZlIiIZE6kriF3/7zQqv0xxCIiIhUgyjDUn5vZdwEPJ5gZRdhNJCIilV+UFsEI4HqCyejzCOYW/nmcQYmISOZEaRGc7O6Xp64ws+8B78UTkoiIZFKUFsF/RlwnIiKVULEtAjM7E/gu0NzMbknZ1JBgDmIREakCSuoaqgXUD8s0SFn/NXBpnEGJiEjmFJsI3P0fwD/MbKK7f5bBmEREJIOiXCz+1szGAh2BOvkr3f2c2KISEZGMiXKxeBKwHGgD/AZYA8yNMSYREcmgKImgqbv/Cdjr7v9w92HAGTHHJSIiGRKla2hv+HODmZ0PrAdaxheSiIhkUpRE8HszawTcSvD8QEPgplijEhGRjCk1Ebj7tPDtNuBsKHiyWEREqoCSHiirDvyYYIyhv7r7YjO7APglUBfompkQRUQkTiW1CP4EtALmAA+b2WfAmcBod38lE8GJiEj8SkoEOUBndz9gZnWALUA7d9+YmdBERCQTSrp9dI+7HwBw913Ax2VNAmbWx8xWmNkqMxtdTJmzzGyBmS0xs3+U5fgiInL4SmoRnGJmi8L3BpwYLhvg7t65pAOH1xgeBc4lmMdgrplNdfelKWUaA48Bfdx9rZkdcxh1ERGRQ1BSIjj1MI/dHVjl7qsBzCwXuAhYmlLmMuBld18L4O6bDvMzRUSkjEoadO5wB5prAaTOdZwH9ChU5iSgppm9TTDC6Th3f67wgcxsODAcoHXr1ocZloiIpIo0ef0hsjTrvNByDeA7wPnAD4E7zeykIju5j3f3HHfPad68eflHKiKSYFGeLD5UeQS3n+ZrSTA8ReEyW9z9G+AbM3sH6AJ8HGNcIiKSIlKLwMzqmtnJZTz2XKC9mbUxs1rAYGBqoTKvAj3NrIaZHUXQdbSsjJ8jIiKHodREYGYXAguAv4bL2WZW+Au9CHffB4wEphN8uf+3uy8xsxFmNiIssyw87iKCB9eedvfFh1oZEREpuyhdQ2MI7gB6G8DdF5hZVpSDu/vrwOuF1j1RaHksMDbK8UREpPxF6Rra5+7bYo9EREQqRJQWwWIzuwyobmbtgVHAzHjDEhGRTInSIriBYL7i3cDzBMNRaz4CEZEqIkqL4GR3/xXwq7iDERGRzIvSInjQzJab2e/MrGPsEYmISEaVmgjc/WzgLGAzMN7MPjKzX8cdmIiIZEakB8rcfaO7PwyMIHim4K5YoxIRkYyJ8kDZqWY2xswWA48Q3DHUMvbIREQkI6JcLH4GmAyc5+6FxwoSEZFKrtRE4O5nZCIQERGpGMUmAjP7b3f/sZl9xMHDR0eaoUxERCqHkloEN4Y/L8hEICIiUjGKvVjs7hvCtz93989SX8DPMxOeiIjELcrto+emWde3vAMREZGKUdI1gp8RnPm3NbNFKZsaAO/FHZiIiGRGSdcIngfeAO4FRqes3+7uX8YalYiIZExJicDdfY2ZXV94g5k1UTIQEakaSmsRXAB8QHD7qKVsc6BtjHGJiEiGFJsI3P2C8GebzIUjIiKZFmWsoe+ZWb3w/U/M7EEzax1/aCIikglRbh99HPjWzLoA/xf4DPivWKMSEZGMiTp5vQMXAePcfRzBLaQiIlIFRBl9dLuZ3QFcAfQ0s+pAzXjDEhGRTInSIhhEMHH9MHffCLQAxsYalYiIZEyUqSo3ApOARmZ2AbDL3Z+LPTIREcmIKHcN/RiYA/wI+DEw28wujTswERHJjCjXCH4FnO7umwDMrDnwv8BLcQYmIiKZEeUaQbX8JBDaGnE/ERGpBKK0CP5qZtMJ5i2G4OLx6/GFJCIimRRlzuJfmNlA4PsE4w2Nd/cpsUcmIiIZUdJ8BO2BB4ATgY+A29x9XaYCExGRzCipr38CMA24hGAE0v8s68HNrI+ZrTCzVWY2uoRyp5vZft2NJCKSeSV1DTVw96fC9yvMbH5ZDhw+gfwowVSXecBcM5vq7kvTlLsfmF6W44uISPkoKRHUMbOu/Gsegrqpy+5eWmLoDqxy99UAZpZLMF7R0kLlbgD+DJxexthFRKQclJQINgAPpixvTFl24JxSjt0C+DxlOQ/okVrAzFoAA8JjFZsIzGw4MBygdWuNgC0iUp5Kmpjm7MM8tqVZ54WW/wO43d33m6UrXhDLeGA8QE5OTuFjiIjIYYjyHMGhygNapSy3BNYXKpMD5IZJoBnQz8z2ufsrMcYlIiIp4kwEc4H2ZtYGWAcMBi5LLZA6DaaZTQSmKQmIiGRWbInA3feZ2UiCu4GqAxPcfYmZjQi3PxHXZ4uISHSlJgIL+m0uB9q6+2/D+Yr/j7vPKW1fd3+dQsNRFJcA3P2qSBGLiEi5ijJ43GPAmcCQcHk7wfMBIiJSBUTpGurh7t3M7EMAd//KzGrFHJeIiGRIlBbB3vDpX4eC+QgOxBqViIhkTJRE8DAwBTjGzP4deBe4J9aoREQkY6IMQz3JzD4AehM8JHaxuy+LPTIREcmIKHcNtQa+BV5LXefua+MMTEREMiPKxeK/EFwfMKAO0AZYAXSMMS4REcmQKF1DnVKXzawbcF1sEYmISEaVeRL6cPhpDRktIlJFRLlGcEvKYjWgG7A5tohERCSjolwjaJDyfh/BNYM/xxOOiIhkWomJIHyQrL67/yJD8YiISIYVe43AzGq4+36CriAREamiSmoRzCFIAgvMbCrwIvBN/kZ3fznm2EREJAOiXCNoAmwlmFc4/3kCB5QIRESqgJISwTHhHUOL+VcCyKd5g0VEqoiSEkF1oD7RJqEXEZFKqqREsMHdf5uxSEREpEKU9GRxupaAiIhUMSUlgt4Zi0JERCpMsYnA3b/MZCAiIlIxyjzonIiIVC1KBCIiCadEICKScEoEIiIJp0QgIpJwSgQiIgmnRCAiknBKBCIiCadEICKScLEmAjPrY2YrzGyVmY1Os/1yM1sUvmaaWZc44xERkaJiSwThfMePAn2BDsAQM+tQqNinwA/cvTPwO2B8XPGIiEh6cbYIugOr3H21u+8BcoGLUgu4+0x3/ypcnAW0jDEeERFJI85E0AL4PGU5L1xXnJ8Cb6TbYGbDzWyemc3bvHlzOYYoIiJxJoLIM5uZ2dkEieD2dNvdfby757h7TvPmzcsxRBERiTJ5/aHKA1qlLLcE1hcuZGadgaeBvu6+NcZ4REQkjThbBHOB9mbWxsxqAYOBqakFzKw18DJwhbt/HGMsIiJSjNhaBO6+z8xGAtOB6sAEd19iZiPC7U8AdwFNgcfMDGCfu+fEFZOIiBQVZ9cQ7v468HqhdU+kvL8GuCbOGEREpGR6slhEJOGUCEREEk6JQEQk4ZQIREQSTolARCThlAhERBJOiUBEJOGUCEREEk6JQEQk4ZQIREQSTolARCThlAhERBJOiUBEJOGUCEREEk6JQEQk4ZQIREQSTolARCThlAhERBJOiUBEJOGUCEREEk6JQEQk4WpUdAAiR5K9e/eSl5fHrl27KjoUkUNSp04dWrZsSc2aNSPvo0QgkiIvL48GDRqQlZWFmVV0OCJl4u5s3bqVvLw82rRpE3k/dQ2JpNi1axdNmzZVEpBKycxo2rRpmVu0SgQihSgJSGV2KP+/SgQiIgmnRCByhDEzrrjiioLlffv20bx5cy644IJS961fvz4Aa9as4fnnny9YP2/ePEaNGlX+waaYOnUq9913X4llJk6cyMiRIwEYM2YMRx11FJs2bSrYnh8/QPXq1cnOzqZLly5069aNmTNnpj3mzp07+cEPfsD+/fsL1j300EPUqVOHbdu2pf3sfGeddRbz5s0DYMeOHVx33XWceOKJdOzYkV69ejF79uyItU/P3Rk1ahTt2rWjc+fOzJ8/P225t956i27dunHaaadx5ZVXsm/fPgC2bdvGhRdeSJcuXejYsSPPPPMMAHv27KFXr14F5Q6XEoHIEaZevXosXryYnTt3AvDmm2/SokWLMh2jcCLIycnh4YcfLtc4C+vfvz+jR48u0z7NmjXjj3/8Y9ptdevWZcGCBSxcuJB7772XO+64I225CRMmMHDgQKpXr16wbvLkyZx++ulMmTIlcizXXHMNTZo0YeXKlSxZsoSJEyeyZcuWMtWnsDfeeIOVK1eycuVKxo8fz89+9rMiZQ4cOMCVV15Jbm4uixcv5oQTTuDZZ58F4NFHH6VDhw4sXLiQt99+m1tvvZU9e/ZQq1YtevfuzQsvvHBY8eXTXUMixfjNa0tYuv7rcj1mh+MbcveFHUst17dvX/7yl79w6aWXMnnyZIYMGcKMGTOA4Ey6fv363HbbbQCcdtppTJs2jaysrIL9R48ezbJly8jOzubKK6+ka9euPPDAA0ybNo0xY8awdu1aVq9ezdq1a7npppsKWgsPPvggEyZMAIIvxptuuok1a9bQp08fvv/97zNr1iy6dOnC1Vdfzd13382mTZuYNGkS3bt3Z+LEicybN49HHnmE1157jd///vfs2bOHpk2bMmnSJI499tgi9Rw2bBgTJ07k9ttvp0mTJsX+Pr7++muOPvrotNsmTZp0UNL75JNP2LFjB2PHjuWee+7hqquuKvX3/cknnzB79mwmTZpEtWrB+XHbtm1p27ZtqfuW5NVXX2Xo0KGYGWeccQb//Oc/2bBhA8cdd1xBma1bt1K7dm1OOukkAM4991zuvfdefvrTn2JmbN++HXdnx44dNGnShBo1gq/tiy++mDvuuIPLL7/8sGIEtQhEjkiDBw8mNzeXXbt2sWjRInr06FGm/e+77z569uzJggULuPnmm4tsX758OdOnT2fOnDn85je/Ye/evXzwwQc888wzzJ49m1mzZvHUU0/x4YcfArBq1SpuvPFGFi1axPLly3n++ed59913eeCBB7jnnnuKHD8/aXz44YcMHjyYP/zhD2njrF+/PsOGDWPcuHFFtu3cuZPs7GxOOeUUrrnmGu68884iZfbs2cPq1asPSoL5ibNnz56sWLHioK6n4ixZsoTs7OyDWhXFGTRoENnZ2UVezz33XJGy69ato1WrVgXLLVu2ZN26dQeVadasGXv37i3oonrppZf4/PPPARg5ciTLli3j+OOPp1OnTowbN64gUZ122mnMnTu31HijUItApBhRztzj0rlzZ9asWcPkyZPp169fuR///PPPp3bt2tSuXZtjjjmGL774gnfffZcBAwZQr149AAYOHMiMGTPo378/bdq0oVOnTgB07NiR3r17Y2Z06tSJNWvWFDl+Xl4egwYNYsOGDezZs6fEe9pHjRpFdnY2t95660Hr87uGAN5//32GDh3K4sWLD7orZsuWLTRu3Pig/XJzc5kyZQrVqlVj4MCBvPjii1x//fXF3k1T1rtsytId4+6lfp6ZkZuby80338zu3bs577zzCs76p0+fTnZ2Nm+99RaffPIJ5557Lj179qRhw4ZUr16dWrVqsX37dho0aFCmOhQWa4vAzPqY2QozW2VmRToPLfBwuH2RmXWLMx6RyqR///7cdtttDBky5KD1NWrU4MCBAwXLh/IUdO3atQveV69enX379qX90kpXvlq1agXL1apVS3vB8oYbbmDkyJF89NFHPPnkkyXG2LhxYy677DIee+yxYsuceeaZbNmyhc2bNx+0vm7dugcde9GiRaxcuZJzzz2XrKwscnNzmTx5MgBNmzblq6++Omj/L7/8kmbNmtGxY0cWLlx40O+1OGVpEbRs2bLg7B6CBHn88cenrd+MGTOYM2cOvXr1on379gA888wzDBw4EDOjXbt2tGnThuXLlxfst3v3burUqVNqzKWJLRGYWXXgUaAv0AEYYmYdChXrC7QPX8OBx+OKR6SyGTZsGHfddVfBmXi+rKysgrtP5s+fz6efflpk3wYNGrB9+/YyfV6vXr145ZVX+Pbbb/nmm2+YMmUKPXv2PKTYt23bVnCBO//CZ0luueUWnnzyyWLvglm+fDn79++nadOmB60/+uij2b9/f0EymDx5MmPGjGHNmjWsWbOG9evXs27dOj777DNOP/103nvvPTZu3AgEd1Lt3r2bVq1aceKJJ5KTk8Pdd99dkBBXrlzJq6++WiSWF154gQULFhR5DR06tEjZ/v3789xzz+HuzJo1i0aNGh10fSBffvfV7t27uf/++xkxYgQArVu35m9/+xsAX3zxBStWrCi4brF161aaN29epqEkihNni6A7sMrdV7v7HiAXuKhQmYuA5zwwC2hsZkV/SyIJ1LJlS2688cYi6y+55BK+/PJLsrOzefzxxwsuMqbq3LkzNWrUoEuXLjz00EORPq9bt25cddVVdO/enR49enDNNdfQtWvXQ4p9zJgx/OhHP6Jnz540a9as1PLNmjVjwIAB7N69u2Bd/jWC7OxsBg0axLPPPpu2D/+8887j3XffBYJuoQEDBhy0fcCAAeTm5nLssccybtw4+vXrR3Z2NjfddBOTJ08u6HN/+umn2bhxI+3ataNTp05ce+21ac/ey6Jfv360bduWdu3ace211x7U6unXrx/r168HYOzYsZx66ql07tyZCy+8kHPOOQeAO++8k5kzZ9KpUyd69+7N/fffX/D7/Pvf/15+3YbuHssLuBR4OmX5CuCRQmWmAd9PWf4bkJPmWMOBecC81q1b+6EYM3Wxj5m6+JD2leRYunRpRYcgZTR//nz/yU9+UtFhZNyAAQN8+fLlabel+z8G5nkx39dxXixOdwWmcCdklDK4+3hgPEBOTk7xHZklqMgLfyISn65du3L22Wezf//+SHf9VAV79uzh4osv5uSTTy6X48WZCPKAVinLLYH1h1BGRKREw4YNq+gQMqpWrVppr0kcqjivEcwF2ptZGzOrBQwGphYqMxUYGt49dAawzd03xBiTSKm8hLtnRI50h/L/G1uLwN33mdlIYDpQHZjg7kvMbES4/QngdaAfsAr4Frg6rnhEoqhTpw5bt27VUNRSKXk4H0FZbym1ynb2k5OT4/lP4ImUN81QJpVdcTOUmdkH7p6Tbh89WSySombNmmWa2UmkKtBYQyIiCadEICKScEoEIiIJV+kuFpvZZuCzQ9y9GXB4M01UPqpzMqjOyXA4dT7B3Zun21DpEsHhMLN5xV01r6pU52RQnZMhrjqra0hEJOGUCEREEi5piWB8RQdQAVTnZFCdkyGWOifqGoGIiBSVtBaBiIgUokQgIpJwVTIRmFkfM1thZqvMbHSa7WZmD4fbF5lZt4qIszxFqPPlYV0XmdlMM+tSEXGWp9LqnFLudDPbb2aXZjK+OESps5mdZWYLzGyJmf0j0zGWtwj/243M7DUzWxjWuVKPYmxmE8xsk5ktLmZ7+X9/FTd1WWV9EQx5/QnQFqgFLAQ6FCrTD3iDYIa0M4DZFR13Bur8XeDo8H3fJNQ5pdxbBEOeX1rRcWfg79wYWAq0DpePqei4M1DnXwL3h++bA18CtSo69sOocy+gG7C4mO3l/v1VFVsE3YFV7r7a3fcAucBFhcpcBDzngVlAYzM7LtOBlqNS6+zuM939q3BxFsFscJVZlL8zwA3An4FNmQwuJlHqfBnwsruvBXD3yl7vKHV2oIEFE0jUJ0gE+zIbZvlx93cI6lCccv/+qoqJoAXwecpyXriurGUqk7LW56cEZxSVWal1NrMWwADgiQzGFacof+eTgKPN7G0z+8DMym8+w4oRpc6PAKcSTHP7EXCjux/ITHgVoty/v6rifATpppUqfI9slDKVSeT6mNnZBIng+7FGFL8odf4P4HZ3319FZhuLUucawHeA3kBd4H0zm+XuH8cdXEyi1PmHwALgHOBE4E0zm+HuX8cdXAUp9++vqpgI8oBWKcstCc4UylqmMolUHzPrDDwN9HX3rRmKLS5R6pwD5IZJoBnQz8z2ufsrmQmx3EX9397i7t8A35jZO0AXoLImgih1vhq4z4MO9FVm9ilwCjAnMyFmXLl/f1XFrqG5QHsza2NmtYDBwNRCZaYCQ8Or72cA29x9Q6YDLUel1tnMWgMvA1dU4rPDVKXW2d3buHuWu2cBLwE/r8RJAKL9b78K9DSzGmZ2FNADWJbhOMtTlDqvJWgBYWbHAicDqzMaZWaV+/dXlWsRuPs+MxsJTCe442CCuy8xsxHh9icI7iDpB6wCviU4o6i0Itb5LqAp8Fh4hrzPK/HIjRHrXKVEqbO7LzOzvwKLgAPA0+6e9jbEyiDi3/l3wEQz+4ig2+R2d6+0w1Ob2WTgLKCZmeUBdwM1Ib7vLw0xISKScFWxa0hERMpAiUBEJOGUCEREEk6JQEQk4ZQIREQSTolAjkjhaKELUl5ZJZTdUQ6fN9HMPg0/a76ZnXkIx3jazDqE739ZaNvMw40xPE7+72VxOOJm41LKZ5tZv/L4bKm6dPuoHJHMbIe71y/vsiUcYyIwzd1fMrPzgAfcvfNhHO+wYyrtuGb2LPCxu/97CeWvAnLcfWR5xyJVh1oEUimYWX0z+1t4tv6RmRUZadTMjjOzd1LOmHuG688zs/fDfV80s9K+oN8B2oX73hIea7GZ3RSuq2dmfwnHv19sZoPC9W+bWY6Z3QfUDeOYFG7bEf58IfUMPWyJXGJm1c1srJnNtWCM+esi/FreJxxszMy6WzDPxIfhz5PDJ3F/CwwKYxkUxj4h/JwP0/0eJYEqeuxtvfRK9wL2EwwktgCYQvAUfMNwWzOCpyrzW7Q7wp+3Ar8K31cHGoRl3wHqhetvB+5K83kTCecrAH4EzCYYvO0joB7B8MZLgK7AJcBTKfs2Cn++TXD2XRBTSpn8GAcAz4bvaxGMIlkXGA78OlxfG5gHtEkT546U+r0I9AmXGwI1wvf/Bvw5fH8V8EjK/vcAPwnfNyYYg6heRf+99arYV5UbYkKqjJ3unp2/YGY1gXvMrBfB0AktgGOBjSn7zAUmhGVfcfcFZvYDoAPwXji0Ri2CM+l0xprZr4HNBCO09gameDCAG2b2MtAT+CvwgJndT9CdNKMM9XoDeNjMagN9gHfcfWfYHdXZ/jWLWiOgPfBpof3rmtkCIAv4AHgzpfyzZtaeYCTKmsV8/nlAfzO7LVyuA7Smco9HJIdJiUAqi8sJZp/6jrvvNbM1BF9iBdz9nTBRnA/8l5mNBb4C3nT3IRE+4xfu/lL+gpn9W7pC7v6xmX2HYLyXe83sf9z9t1Eq4e67zOxtgqGTBwGT8z8OuMHdp5dyiJ3unm1mjYBpwPXAwwTj7fzd3QeEF9bfLmZ/Ay5x9xVR4pVk0DUCqSwaAZvCJHA2cELhAmZ2QljmKeBPBNP9zQK+Z2b5ff5HmbMlPiIAAAESSURBVNlJET/zHeDicJ96BN06M8zseOBbd/9/wAPh5xS2N2yZpJNLMFBYT4LB1Ah//ix/HzM7KfzMtNx9GzAKuC3cpxGwLtx8VUrR7QRdZPmmAzdY2Dwys67FfYYkhxKBVBaTgBwzm0fQOliepsxZwAIz+5CgH3+cu28m+GKcbGaLCBLDKVE+0N3nE1w7mENwzeBpd/8Q6ATMCbtofgX8Ps3u44FF+ReLC/kfgnlp/9eD6RchmCdiKTDfgknLn6SUFnsYy0KCoZn/QNA6eY/g+kG+vwMd8i8WE7QcaoaxLQ6XJeF0+6iISMKpRSAiknBKBCIiCadEICKScEoEIiIJp0QgIpJwSgQiIgmnRCAiknD/H507tifycfG1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_roc_curve(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}