NLP_Ensemble.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LTTHpwfQm4f3"
      },
      "source": [
        "# NLP Ensemble approach\n",
        "By Group 16\n",
        "\n",
        "First Level: Multinomial Bayes model <br>\n",
        "Second level: Support Vector Machine\n",
        "\n",
        "I first experimented with how to feed the data into the first level of models.  I tried to see if the BOW MultinomialNB models or TF-IDF MultinomialNB models performed better.  I also tried to see if processing the words lead to better results.\n",
        "\n",
        "At the second level, I tried Random Forest Classification, Support Vector Machine and Logistic Regression.  I went with  SVM in the end.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 93,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f3JwNT9vXna7",
        "outputId": "73bd5b12-1aee-4e13-ffd8-e782df5876e4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "import string\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.base import TransformerMixin\n",
        "from sklearn.metrics import accuracy_score, plot_confusion_matrix, classification_report, confusion_matrix\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, plot_roc_curve,roc_curve\n",
        "import spacy\n",
        "from spacy.lang.en.stop_words import STOP_WORDS\n",
        "from spacy.lang.en import English\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import wordnet, stopwords"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vjVz5We-gI3O",
        "outputId": "98df477d-621f-406a-82b9-48e3e6058f71"
      },
      "outputs": [],
      "source": [
        "df=pd.read_csv('fake_job_postings.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {
        "id": "jQDfX1bsYRZv"
      },
      "outputs": [],
      "source": [
        "df.fillna('empty',inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {
        "id": "GYntgxJxgd2_"
      },
      "outputs": [],
      "source": [
        "columns=['job_id','telecommuting','has_company_logo','has_questions', 'salary_range','employment_type']\n",
        "#remove columns not required for next steps \n",
        "for colu in columns:\n",
        "    del df[colu]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 97,
      "metadata": {
        "id": "wc48fkuIv_5r"
      },
      "outputs": [],
      "source": [
        "#df.head()\n",
        "X = df[[\"title\", \"company_profile\", \"description\", \"requirements\",\"benefits\"]]\n",
        "Y = df[\"fraudulent\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 98,
      "metadata": {
        "id": "eJMdwzIpv7d7"
      },
      "outputs": [],
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X,Y, test_size=0.1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "MC-JXotRgh4m"
      },
      "outputs": [],
      "source": [
        "X_train_title = X_train['title']\n",
        "X_test_title = X_test['title']\n",
        "\n",
        "X_train_profile = X_train[\"company_profile\"]\n",
        "X_test_profile = X_test[\"company_profile\"]\n",
        "\n",
        "X_train_description = X_train[\"description\"]\n",
        "X_test_description = X_test[\"description\"]\n",
        "\n",
        "X_train_requirements = X_train[\"requirements\"]\n",
        "X_test_requirements = X_test[\"requirements\"]\n",
        "\n",
        "X_train_benefits = X_train[\"benefits\"]\n",
        "X_test_benefits = X_test[\"benefits\"]\n",
        "\n",
        "features = {'title':(X_train_title, X_test_title),\n",
        "            'profile':(X_train_profile, X_test_profile),\n",
        "            'description':(X_train_description, X_test_description),\n",
        "            'requirements':(X_train_requirements, X_test_requirements),\n",
        "            'benefits':(X_train_benefits, X_test_benefits)}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sl1tB-Yy9nKM"
      },
      "source": [
        "## Bag of Words MultinomialNB Models\n",
        "\n",
        "I used a MultinomialNB model after fitting in a BOW matrix for each column."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 100,
      "metadata": {
        "id": "mfIl9YysyU_b"
      },
      "outputs": [],
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "def tokenize_test(X_train, y_train, X_test, y_test, vect):\n",
        "    X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "    X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "    # create document-term matrices using the vectorizer\n",
        "    X_train_dtm = vect.fit_transform(X_train)\n",
        "    X_test_dtm = vect.transform(X_test)\n",
        "    \n",
        "    # print the number of features that were generated\n",
        "    print('Features: ', X_train_dtm.shape[1])\n",
        "    \n",
        "    # use Multinomial Naive Bayes to predict the star rating\n",
        "    nb = MultinomialNB()\n",
        "\n",
        "    #RFC\n",
        "    #nb = RandomForestClassifier(n_jobs=3,oob_score=True,n_estimators=100, criterion=\"entropy\")\n",
        "    \n",
        "    nb.fit(X_train_dtm, y_train)\n",
        "    y_pred_class = nb.predict(X_test_dtm)\n",
        "    \n",
        "    # Get the training accuracy\n",
        "    print('Training Accuracy: ', metrics.accuracy_score(y_train, nb.predict(X_train_dtm)))\n",
        "    print(\"--------Test Metrics--------\")\n",
        "    # print the accuracy of its predictions\n",
        "    print('Accuracy Score:', metrics.accuracy_score(y_test, y_pred_class))\n",
        "    print('Precision Score:', (metrics.precision_score(y_test, y_pred_class)))\n",
        "    print('Recall Score:', (metrics.recall_score(y_test, y_pred_class)))\n",
        "    print('F1 Score:', (metrics.f1_score(y_test, y_pred_class)))\n",
        "\n",
        "    y_score_mnb = nb.predict_proba(X_test_dtm)\n",
        "    print('ROC Score:', (metrics.roc_auc_score(y_test, y_score_mnb[:,1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 101,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FRUVN2tZzG1T",
        "outputId": "e1cdb38f-e36e-4ce5-dda3-9410283592d5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "column :  TITLE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  44209\n",
            "Training Accuracy:  0.9674993785732041\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9748322147651006\n",
            "Precision Score: 0.8823529411764706\n",
            "Recall Score: 0.4225352112676056\n",
            "F1 Score: 0.5714285714285715\n",
            "ROC Score: 0.92273208265317\n",
            "\n",
            "\n",
            "column :  PROFILE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  314867\n",
            "Training Accuracy:  0.8693760874968929\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.8680089485458613\n",
            "Precision Score: 0.06806282722513089\n",
            "Recall Score: 0.18309859154929578\n",
            "F1 Score: 0.09923664122137406\n",
            "ROC Score: 0.8376918470637453\n",
            "\n",
            "\n",
            "column :  DESCRIPTION\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  2688494\n",
            "Training Accuracy:  0.9928535918468805\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9821029082774049\n",
            "Precision Score: 0.8979591836734694\n",
            "Recall Score: 0.6197183098591549\n",
            "F1 Score: 0.7333333333333333\n",
            "ROC Score: 0.9029629143527442\n",
            "\n",
            "\n",
            "column :  REQUIREMENTS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  1426265\n",
            "Training Accuracy:  0.9848993288590604\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9793064876957495\n",
            "Precision Score: 0.9722222222222222\n",
            "Recall Score: 0.49295774647887325\n",
            "F1 Score: 0.6542056074766356\n",
            "ROC Score: 0.8539665482704027\n",
            "\n",
            "\n",
            "column :  BENEFITS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1032085433.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  293606\n",
            "Training Accuracy:  0.9174123788217748\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9630872483221476\n",
            "Precision Score: 0.5409836065573771\n",
            "Recall Score: 0.4647887323943662\n",
            "F1 Score: 0.5\n",
            "ROC Score: 0.8322491735503293\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "for key, values in features.items():\n",
        "  print(\"column : \", key.upper())\n",
        "  tokenize_test(values[0], y_train, values[1], y_test, CountVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english', max_df=0.5))\n",
        "  print(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VBpP7bxyMMXU"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xl109pH34tKT"
      },
      "source": [
        "Some results: the description column seems to have a high f1 score.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RQHuuthOoq_b"
      },
      "source": [
        "## TFIDF MultinomialNB Models\n",
        "The cleaned data is fitted by a tf-idf vectoriser and fed into MultinomialNB models."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 102,
      "metadata": {
        "id": "Ch9MLDnE-fik"
      },
      "outputs": [],
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "def tf_idf_test(X_train, y_train, X_test, y_test, vect):\n",
        "    X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "    X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "    # create tf-idf matrices using the vectorizer\n",
        "    X_train_dtm = vect.fit_transform(X_train)\n",
        "    X_test_dtm = vect.transform(X_test)\n",
        "    \n",
        "    # print the number of features that were generated\n",
        "    print('Features: ', X_train_dtm.shape[1])\n",
        "    \n",
        "    # use Multinomial Naive Bayes to predict the star rating\n",
        "    nb = MultinomialNB()\n",
        "\n",
        "    \n",
        "    # #RFC\n",
        "    # nb = RandomForestClassifier(n_jobs=3,oob_score=True,n_estimators=100, criterion=\"entropy\")\n",
        "\n",
        "    nb.fit(X_train_dtm, y_train)\n",
        "    y_pred_class = nb.predict(X_test_dtm)\n",
        "    \n",
        "    # Get the training accuracy\n",
        "    print('Training Accuracy: ', metrics.accuracy_score(y_train, nb.predict(X_train_dtm)))\n",
        "    print(\"--------Test Metrics--------\")\n",
        "    # print the accuracy of its predictions\n",
        "    print('Accuracy Score:', metrics.accuracy_score(y_test, y_pred_class))\n",
        "    print('Precision Score:', (metrics.precision_score(y_test, y_pred_class)))\n",
        "    print('Recall Score:', (metrics.recall_score(y_test, y_pred_class)))\n",
        "    print('F1 Score:', (metrics.f1_score(y_test, y_pred_class)))\n",
        "\n",
        "    y_score_mnb = nb.predict_proba(X_test_dtm)\n",
        "    print('ROC Score:', (metrics.roc_auc_score(y_test, y_score_mnb[:,1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 103,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Shl7-IRnhm34",
        "outputId": "7d5f2b33-fba1-492d-c70b-d77e67fe06fc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "column :  TITLE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  44209\n",
            "Training Accuracy:  0.9590479741486453\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9681208053691275\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.19718309859154928\n",
            "F1 Score: 0.32941176470588235\n",
            "ROC Score: 0.8962897946795507\n",
            "\n",
            "\n",
            "column :  PROFILE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  314867\n",
            "Training Accuracy:  0.9587994034302759\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9574944071588367\n",
            "Precision Score: 0.4074074074074074\n",
            "Recall Score: 0.15492957746478872\n",
            "F1 Score: 0.22448979591836735\n",
            "ROC Score: 0.8714183763032476\n",
            "\n",
            "\n",
            "column :  DESCRIPTION\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  2688494\n",
            "Training Accuracy:  0.9622172508078548\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9714765100671141\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.28169014084507044\n",
            "F1 Score: 0.43956043956043955\n",
            "ROC Score: 0.8463911014133724\n",
            "\n",
            "\n",
            "column :  REQUIREMENTS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  1426266\n",
            "Training Accuracy:  0.9582401193139448\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9697986577181208\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.23943661971830985\n",
            "F1 Score: 0.38636363636363635\n",
            "ROC Score: 0.8021729679181671\n",
            "\n",
            "\n",
            "column :  BENEFITS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\2547024235.py:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  293606\n",
            "Training Accuracy:  0.9654486701466567\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9720357941834452\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.29577464788732394\n",
            "F1 Score: 0.45652173913043476\n",
            "ROC Score: 0.836686982699927\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn import metrics\n",
        "\n",
        "for key, values in features.items():\n",
        "  print(\"column : \", key.upper())\n",
        "  tf_idf_test(values[0], y_train, values[1], y_test, TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')) #, max_df=0.3\n",
        "  print(\"\\n\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OK8-93nQd_5y"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MJA6-Fr3N6y5"
      },
      "source": [
        "##Processed BOW in MultinomialNB models\n",
        "This section includes steps in stemming and lemmatization before feeding the data into the count vectorizer.  \n",
        "\n",
        "The results are mostly quite similar to unprocessed data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "id": "H5xYbRr4NINB"
      },
      "outputs": [],
      "source": [
        "from nltk.stem.snowball import SnowballStemmer\n",
        "stemmer = SnowballStemmer(language='english')\n",
        "revisions = [stemmer.stem(word) for word in X_train_title]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 105,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XeWE81HCNbza",
        "outputId": "e329639f-e171-4863-809b-2d2d05f7505c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Original                                         Stemmed\n",
            "-----------------------------------------------  ----------------------------------------------\n",
            "Lawn and Maintenance Contractors                 lawn and maintenance contractors\n",
            "Digital Procurement Officer                      digital procurement offic\n",
            "International Broadcaster, Azerbaijani Language  international broadcaster, azerbaijani languag\n",
            "Head of Sales - B2B                              head of sales - b2b\n",
            "Sales Consultant                                 sales consult\n"
          ]
        }
      ],
      "source": [
        "#revisions\n",
        "# pip install tabulate\n",
        "from tabulate import tabulate\n",
        "print (tabulate({\"Original\": X_train_title[0:5],\"Stemmed\": revisions[0:5]}, headers=\"keys\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 106,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GGUSU7d0ODjg",
        "outputId": "40068f47-b8f5-4764-cab2-32aaeb01327c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package omw-1.4 to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('wordnet')\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "\n",
        "def tokenize_test_stemmed(X_train, y_train, X_test, y_test, vect):\n",
        "    X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "    X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "    #Stemming\n",
        "    stemmer = SnowballStemmer(language='english')\n",
        "    X_train = [stemmer.stem(word) for word in X_train]\n",
        "    X_test = [stemmer.stem(word) for word in X_test]\n",
        "\n",
        "    #Lemmatization\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    X_train = [lemmatizer.lemmatize(word) for word in X_train]\n",
        "    X_test = [lemmatizer.lemmatize(word) for word in X_test]\n",
        "\n",
        "    # create document-term matrices using the vectorizer\n",
        "    X_train_dtm = vect.fit_transform(X_train)\n",
        "    X_test_dtm = vect.transform(X_test)\n",
        "\n",
        "    # stemmer = SnowballStemmer(language='english')\n",
        "    # X_train_dtm = [stemmer.stem(word) for word in X_train_dtm]\n",
        "    # X_test_dtm = [stemmer.stem(word) for word in X_train_dtm]\n",
        "    \n",
        "    # print the number of features that were generated\n",
        "    print('Features: ', X_train_dtm.shape[1])\n",
        "    \n",
        "    # use Multinomial Naive Bayes to predict the star rating\n",
        "    nb = MultinomialNB()\n",
        "\n",
        "    \n",
        "    #RFC\n",
        "    #nb = RandomForestClassifier(n_jobs=3,oob_score=True,n_estimators=100, criterion=\"entropy\")\n",
        "\n",
        "    nb.fit(X_train_dtm, y_train)\n",
        "    y_pred_class = nb.predict(X_test_dtm)\n",
        "    \n",
        "    # Get the training accuracy\n",
        "    print('Training Accuracy: ', metrics.accuracy_score(y_train, nb.predict(X_train_dtm)))\n",
        "    print(\"--------Test Metrics--------\")\n",
        "    # print the accuracy of its predictions\n",
        "    print('Accuracy Score:', metrics.accuracy_score(y_test, y_pred_class))\n",
        "    print('Precision Score:', (metrics.precision_score(y_test, y_pred_class)))\n",
        "    print('Recall Score:', (metrics.recall_score(y_test, y_pred_class)))\n",
        "    print('F1 Score:', (metrics.f1_score(y_test, y_pred_class)))\n",
        "\n",
        "    y_score_mnb = nb.predict_proba(X_test_dtm)\n",
        "    print('ROC Score:', (metrics.roc_auc_score(y_test, y_score_mnb[:,1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 107,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RcNVgjnLOYLR",
        "outputId": "ac7fc132-766b-4a72-fdae-4bc0e9df6215"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "column :  TITLE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  41884\n",
            "Training Accuracy:  0.9696743723589362\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9742729306487695\n",
            "Precision Score: 0.8787878787878788\n",
            "Recall Score: 0.4084507042253521\n",
            "F1 Score: 0.5576923076923077\n",
            "ROC Score: 0.9138974792259673\n",
            "\n",
            "\n",
            "column :  PROFILE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  299560\n",
            "Training Accuracy:  0.870308227690778\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.8708053691275168\n",
            "Precision Score: 0.06989247311827956\n",
            "Recall Score: 0.18309859154929578\n",
            "F1 Score: 0.10116731517509725\n",
            "ROC Score: 0.8396974743041827\n",
            "\n",
            "\n",
            "column :  DESCRIPTION\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  2579588\n",
            "Training Accuracy:  0.9937235893611732\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9821029082774049\n",
            "Precision Score: 0.8979591836734694\n",
            "Recall Score: 0.6197183098591549\n",
            "F1 Score: 0.7333333333333333\n",
            "ROC Score: 0.9033402511750761\n",
            "\n",
            "\n",
            "column :  REQUIREMENTS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  1372779\n",
            "Training Accuracy:  0.9844021874223217\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9787472035794184\n",
            "Precision Score: 0.9714285714285714\n",
            "Recall Score: 0.4788732394366197\n",
            "F1 Score: 0.6415094339622641\n",
            "ROC Score: 0.8559762769980395\n",
            "\n",
            "\n",
            "column :  BENEFITS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\1722342225.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  277467\n",
            "Training Accuracy:  0.9200223713646533\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9608501118568232\n",
            "Precision Score: 0.5081967213114754\n",
            "Recall Score: 0.43661971830985913\n",
            "F1 Score: 0.46969696969696967\n",
            "ROC Score: 0.8213064057027077\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "for key, values in features.items():\n",
        "  print(\"column : \", key.upper())\n",
        "  tokenize_test_stemmed(values[0], y_train, values[1], y_test, CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5))  #, max_df = 0.5\n",
        "  print(\"\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SVunPqt0bQsn"
      },
      "source": [
        "##Processed TF-IDF in Multinomial Models\n",
        "\n",
        "The textual columns were cleaned, tokenized, stemmed, lemmatized etc and fed into MultinomialNB models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 108,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M-7f9QysSV9I",
        "outputId": "9fb33c6c-e82f-422d-caf3-9375703ec0c0"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package omw-1.4 to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "import nltk\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('wordnet')\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "\n",
        "def tf_idf_test_processed(X_train, y_train, X_test, y_test, vect):\n",
        "    X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "    X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "    #Stemming\n",
        "    stemmer = SnowballStemmer(language='english')\n",
        "    X_train = [stemmer.stem(word) for word in X_train]\n",
        "    X_test = [stemmer.stem(word) for word in X_test]\n",
        "\n",
        "    #Lemmatization\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    X_train = [lemmatizer.lemmatize(word) for word in X_train]\n",
        "    X_test = [lemmatizer.lemmatize(word) for word in X_test]\n",
        "\n",
        "    # create tf-idf matrices using the vectorizer\n",
        "    X_train_dtm = vect.fit_transform(X_train)\n",
        "    X_test_dtm = vect.transform(X_test)\n",
        "    \n",
        "    # print the number of features that were generated\n",
        "    print('Features: ', X_train_dtm.shape[1])\n",
        "    \n",
        "    # use Multinomial Naive Bayes to predict the star rating\n",
        "    nb = MultinomialNB()\n",
        "\n",
        "    \n",
        "    # #RFC\n",
        "    # nb = RandomForestClassifier(n_jobs=3,oob_score=True,n_estimators=100, criterion=\"entropy\")\n",
        "\n",
        "    nb.fit(X_train_dtm, y_train)\n",
        "    y_pred_class = nb.predict(X_test_dtm)\n",
        "    \n",
        "    # Get the training accuracy\n",
        "    print('Training Accuracy: ', metrics.accuracy_score(y_train, nb.predict(X_train_dtm)))\n",
        "    print(\"--------Test Metrics--------\")\n",
        "    # print the accuracy of its predictions\n",
        "    print('Accuracy Score:', metrics.accuracy_score(y_test, y_pred_class))\n",
        "    print('Precision Score:', (metrics.precision_score(y_test, y_pred_class)))\n",
        "    print('Recall Score:', (metrics.recall_score(y_test, y_pred_class)))\n",
        "    print('F1 Score:', (metrics.f1_score(y_test, y_pred_class)))\n",
        "\n",
        "    y_score_mnb = nb.predict_proba(X_test_dtm)\n",
        "    print('ROC Score:', (metrics.roc_auc_score(y_test, y_score_mnb[:,1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 109,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2JoqBGCSRKI",
        "outputId": "59b39202-54e4-4d45-8c49-b69a634f186b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "column :  TITLE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  41884\n",
            "Training Accuracy:  0.9586751180710912\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9681208053691275\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.19718309859154928\n",
            "F1 Score: 0.32941176470588235\n",
            "ROC Score: 0.8820863445085187\n",
            "\n",
            "\n",
            "column :  PROFILE\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  299560\n",
            "Training Accuracy:  0.9602286850608999\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.959731543624161\n",
            "Precision Score: 0.4782608695652174\n",
            "Recall Score: 0.15492957746478872\n",
            "F1 Score: 0.23404255319148937\n",
            "ROC Score: 0.8444880113529165\n",
            "\n",
            "\n",
            "column :  DESCRIPTION\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  2579588\n",
            "Training Accuracy:  0.960974397216008\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.970917225950783\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.2676056338028169\n",
            "F1 Score: 0.42222222222222217\n",
            "ROC Score: 0.8462270419254022\n",
            "\n",
            "\n",
            "column :  REQUIREMENTS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  1372780\n",
            "Training Accuracy:  0.9582401193139448\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9697986577181208\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.23943661971830985\n",
            "F1 Score: 0.38636363636363635\n",
            "ROC Score: 0.8263061185986038\n",
            "\n",
            "\n",
            "column :  BENEFITS\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:9: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train = X_train.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\875048184.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test = X_test.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Features:  277467\n",
            "Training Accuracy:  0.9645165299527716\n",
            "--------Test Metrics--------\n",
            "Accuracy Score: 0.9720357941834452\n",
            "Precision Score: 1.0\n",
            "Recall Score: 0.29577464788732394\n",
            "F1 Score: 0.45652173913043476\n",
            "ROC Score: 0.7748652661455042\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn import metrics\n",
        "\n",
        "for key, values in features.items():\n",
        "  print(\"column : \", key.upper())\n",
        "  tf_idf_test_processed(values[0], y_train, values[1], y_test, TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english'))\n",
        "  print(\"\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rj4I7gY9TJa5"
      },
      "source": [
        "##Findings from tuning\n",
        "There is no significant difference in model performance whether the data is processed or not.  However, the inclusion of unigram, bigram, trigram and quadgrams greatly improve f1 score.  This would imply that certain phrases are highly predictive of fraud entries."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7b9tzklvfJ6K"
      },
      "source": [
        "#Ensemble Learning first level\n",
        "First, I cleaned the data, removed stop words, stemmed, lemmatized the data using the NLTK package.  Then I created countVectorizers objects to tokenize the data.  This can also help me retrieve feature importance later.\n",
        "\n",
        "I obtain 5 BOW matrices and 5 TF-IDF matrices from the 5 columns. Then, I fed each matrix into a MultinomialNB model each to give me 10 columns of predicted results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 110,
      "metadata": {
        "id": "cCX2zjaef-1S"
      },
      "outputs": [],
      "source": [
        "cv_title = CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5)\n",
        "cv_profile = CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5)\n",
        "cv_description = CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5)\n",
        "cv_requirement = CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5)\n",
        "cv_benefit = CountVectorizer(lowercase=True, ngram_range=(1, 4), stop_words='english', max_df=0.5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 111,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZYf5or5-hc16",
        "outputId": "47ff41da-6d41-4119-830d-274278ac6533"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package omw-1.4 to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to\n",
            "[nltk_data]     C:\\Users\\hwhme\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:10: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train_title = X_train_title.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:11: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test_title = X_test_title.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:13: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train_profile = X_train_profile.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:14: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test_profile = X_test_profile.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:16: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train_description = X_train_description.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:17: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test_description = X_test_description.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:19: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train_requirements = X_train_requirements.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:20: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test_requirements = X_test_requirements.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:22: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_train_benefits = X_train_benefits.str.replace('[^a-zA-Z0-9 ]', '')\n",
            "C:\\Users\\hwhme\\AppData\\Local\\Temp\\ipykernel_4556\\3034138221.py:23: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  X_test_benefits = X_test_benefits.str.replace('[^a-zA-Z0-9 ]', '')\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('wordnet')\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "\n",
        "stemmer = SnowballStemmer(language='english')\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "X_train_title = X_train_title.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "X_test_title = X_test_title.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "X_train_profile = X_train_profile.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "X_test_profile = X_test_profile.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "X_train_description = X_train_description.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "X_test_description = X_test_description.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "X_train_requirements = X_train_requirements.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "X_test_requirements = X_test_requirements.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "X_train_benefits = X_train_benefits.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "X_test_benefits = X_test_benefits.str.replace('[^a-zA-Z0-9 ]', '')\n",
        "\n",
        "#stemming\n",
        "X_train_title_pro = [stemmer.stem(word) for word in X_train_title]\n",
        "X_test_title_pro = [stemmer.stem(word) for word in X_test_title]\n",
        "\n",
        "X_train_profile_pro = [stemmer.stem(word) for word in X_train_profile]\n",
        "X_test_profile_pro = [stemmer.stem(word) for word in X_test_profile]\n",
        "\n",
        "X_train_description_pro = [stemmer.stem(word) for word in X_train_description]\n",
        "X_test_description_pro = [stemmer.stem(word) for word in X_test_description]\n",
        "\n",
        "X_train_requirements_pro = [stemmer.stem(word) for word in X_train_requirements]\n",
        "X_test_requirements_pro = [stemmer.stem(word) for word in X_test_requirements]\n",
        "\n",
        "X_train_benefits_pro = [stemmer.stem(word) for word in X_train_benefits]\n",
        "X_test_benefits_pro = [stemmer.stem(word) for word in X_test_benefits]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 112,
      "metadata": {
        "id": "DF3PO1YzidtM"
      },
      "outputs": [],
      "source": [
        "#lemmatization\n",
        "X_train_title_pro = [lemmatizer.lemmatize(word) for word in X_train_title]\n",
        "X_test_title_pro = [lemmatizer.lemmatize(word) for word in X_test_title]\n",
        "\n",
        "X_train_profile_pro = [lemmatizer.lemmatize(word) for word in X_train_profile]\n",
        "X_test_profile_pro = [lemmatizer.lemmatize(word) for word in X_test_profile]\n",
        "\n",
        "X_train_description_pro = [lemmatizer.lemmatize(word) for word in X_train_description]\n",
        "X_test_description_pro = [lemmatizer.lemmatize(word) for word in X_test_description]\n",
        "\n",
        "X_train_requirements_pro = [lemmatizer.lemmatize(word) for word in X_train_requirements]\n",
        "X_test_requirements_pro = [lemmatizer.lemmatize(word) for word in X_test_requirements]\n",
        "\n",
        "X_train_benefits_pro = [lemmatizer.lemmatize(word) for word in X_train_benefits]\n",
        "X_test_benefits_pro = [lemmatizer.lemmatize(word) for word in X_test_benefits]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 113,
      "metadata": {
        "id": "sO3slw85itai"
      },
      "outputs": [],
      "source": [
        "features_pro = {'title':(X_train_title_pro, X_test_title_pro),\n",
        "            'profile':(X_train_profile_pro, X_test_profile_pro),\n",
        "            'description':(X_train_description_pro, X_test_description_pro),\n",
        "            'requirements':(X_train_requirements_pro, X_test_requirements_pro),\n",
        "            'benefits':(X_train_benefits_pro, X_test_benefits_pro)}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 114,
      "metadata": {
        "id": "cxlhyAo8jbTa"
      },
      "outputs": [],
      "source": [
        "nb_title = MultinomialNB()\n",
        "nb_profile = MultinomialNB()\n",
        "nb_description = MultinomialNB()\n",
        "nb_requirements = MultinomialNB()\n",
        "nb_benefits = MultinomialNB()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 115,
      "metadata": {
        "id": "eGwvJCAnnmez"
      },
      "outputs": [],
      "source": [
        "X_train_title_bow = cv_title.fit_transform(X_train_title_pro)\n",
        "X_test_title_bow = cv_title.transform(X_test_title_pro)\n",
        "\n",
        "X_train_profile_bow = cv_profile.fit_transform(X_train_profile_pro)\n",
        "X_test_profile_bow = cv_profile.transform(X_test_profile_pro)\n",
        "\n",
        "X_train_description_bow = cv_description.fit_transform(X_train_description_pro)\n",
        "X_test_description_bow = cv_description.transform(X_test_description_pro)\n",
        "\n",
        "X_train_requirements_bow = cv_requirement.fit_transform(X_train_requirements_pro)\n",
        "X_test_requirements_bow = cv_requirement.transform(X_test_requirements_pro)\n",
        "\n",
        "X_train_benefits_bow = cv_benefit.fit_transform(X_train_benefits_pro)\n",
        "X_test_benefits_bow = cv_benefit.transform(X_test_benefits_pro)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 116,
      "metadata": {
        "id": "rp3OD0EGjS8D"
      },
      "outputs": [],
      "source": [
        "# defining parameter range \n",
        "from sklearn.model_selection import GridSearchCV\n",
        "nb_param_grid = {'alpha': [1, 0, 0.1, 0.01, 0.001],  \n",
        "              'fit_prior': [False, True], \n",
        "              }  \n",
        "   \n",
        "nb_title_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "nb_profile_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "nb_description_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "nb_requirements_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "nb_benefits_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 117,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vnRw70YJjTFx",
        "outputId": "c1c884ca-c582-410b-83ad-033e1e1a28b3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.833 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.812 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.848 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.833 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.812 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.848 total time=   0.0s\n",
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.887 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.879 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.882 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.888 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.879 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.882 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.847 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.833 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.855 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.847 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.833 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.856 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.865 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.850 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.871 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.865 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.850 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.871 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.879 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.864 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.882 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.879 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.864 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.882 total time=   0.0s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.838 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.845 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.836 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.839 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.846 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.836 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.940 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.939 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.936 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.940 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.939 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.936 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.867 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.876 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.867 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.868 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.876 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.867 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.905 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.911 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.903 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.905 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.911 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.903 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.906 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.911 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.905 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.906 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.911 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.905 total time=   0.0s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.916 total time=   0.1s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.919 total time=   0.1s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.930 total time=   0.1s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.919 total time=   0.1s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.921 total time=   0.1s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.932 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.940 total time=   0.1s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.933 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.939 total time=   0.1s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.940 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.933 total time=   0.1s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.940 total time=   0.1s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.747 total time=   0.1s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.746 total time=   0.1s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.741 total time=   0.1s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.747 total time=   0.1s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.748 total time=   0.1s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.743 total time=   0.1s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.760 total time=   0.1s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.758 total time=   0.1s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.754 total time=   0.1s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.760 total time=   0.1s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.759 total time=   0.1s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.755 total time=   0.1s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.788 total time=   0.1s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.786 total time=   0.1s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.785 total time=   0.1s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.789 total time=   0.1s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.788 total time=   0.1s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.786 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.845 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.821 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.855 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.845 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.822 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.855 total time=   0.0s\n",
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.884 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.867 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.889 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.884 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.867 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.889 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.653 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.680 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.688 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.654 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.680 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.689 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.672 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.689 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.700 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.671 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.690 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.701 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.700 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.717 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.732 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.700 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.719 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.733 total time=   0.0s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.741 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.793 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.789 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.741 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.793 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.790 total time=   0.0s\n",
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.790 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.853 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.851 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.790 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.853 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.851 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.744 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.794 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.785 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.744 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.795 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.786 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.761 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.805 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.802 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.761 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.806 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.803 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.770 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.816 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.815 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.771 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.816 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.815 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# fitting the model for grid search \n",
        "nb_title_tuned = nb_title_tuned.fit(X_train_title_bow, y_train)\n",
        "nb_profile_tuned = nb_profile_tuned.fit(X_train_profile_bow, y_train)\n",
        "nb_description_tuned = nb_description_tuned.fit(X_train_description_bow, y_train)\n",
        "nb_requirements_tuned = nb_requirements_tuned.fit(X_train_requirements_bow, y_train)\n",
        "nb_benefits_tuned = nb_benefits_tuned.fit(X_train_benefits_bow, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q6d_OLfJ2M-S"
      },
      "source": [
        "This gives the best multinomialNB parameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 118,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bdUf8MSBjTH5",
        "outputId": "02097a63-613c-422f-934f-c2bfeb904da9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'alpha': 0, 'fit_prior': True}\n",
            "{'alpha': 0, 'fit_prior': True}\n",
            "{'alpha': 0, 'fit_prior': True}\n",
            "{'alpha': 0, 'fit_prior': True}\n",
            "{'alpha': 0, 'fit_prior': True}\n"
          ]
        }
      ],
      "source": [
        "# print best parameter after tuning \n",
        "print(nb_title_tuned.best_params_) \n",
        "print(nb_profile_tuned.best_params_) \n",
        "print(nb_description_tuned.best_params_) \n",
        "print(nb_requirements_tuned.best_params_) \n",
        "print(nb_benefits_tuned.best_params_)\n",
        "\n",
        "#result:\n",
        "# {'alpha': 0, 'fit_prior': True}\n",
        "# {'alpha': 0, 'fit_prior': True}\n",
        "# {'alpha': 0, 'fit_prior': True}\n",
        "# {'alpha': 0, 'fit_prior': True}\n",
        "# {'alpha': 0, 'fit_prior': True}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 119,
      "metadata": {
        "id": "cjhtO7gmk6fa"
      },
      "outputs": [],
      "source": [
        "nb_title_train_pred = nb_title_tuned.predict(X_train_title_bow)\n",
        "nb_profile_train_pred = nb_profile_tuned.predict(X_train_profile_bow)\n",
        "nb_description_train_pred = nb_description_tuned.predict(X_train_description_bow)\n",
        "nb_requirements_train_pred = nb_requirements_tuned.predict(X_train_requirements_bow)\n",
        "nb_benefits_train_pred = nb_benefits_tuned.predict(X_train_benefits_bow)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 120,
      "metadata": {
        "id": "dwXfTe45sDGd"
      },
      "outputs": [],
      "source": [
        "X_train_pred = [nb_title_train_pred, nb_profile_train_pred, nb_description_train_pred, nb_description_train_pred, nb_benefits_train_pred]\n",
        "X_train_pred = np.array(X_train_pred)\n",
        "X_train_pred = X_train_pred.T"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 121,
      "metadata": {
        "id": "kRyiIFmHsr6d"
      },
      "outputs": [],
      "source": [
        "rfc = RandomForestClassifier(oob_score = True)\n",
        "result = rfc.fit(X_train_pred,y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 122,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "xQ2ncIACs5Zj",
        "outputId": "72d48a0b-e4db-4fbb-c69a-ca96b7b6da5a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9995650012428536\n",
            "Precision Score: 0.993734335839599\n",
            "Recall Score: 0.9974842767295597\n",
            "F1 Score: 0.9956057752667922\n",
            "ROC Score: 0.9993014209634561\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A0A73669E0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqD0lEQVR4nO3df5xVVb3/8ddbRAFBSEGviQQGyS9llElC0iArDQXLDMQ0Sb3e/JHeVEzTq4ZlmUYl6jVMRA0BKS38fb/lzxR/gIzID38gQgxaIioXLhIon+8fe890ZjjM2cPMOeMw7+fjMY85e++19/6sfWbOOmutvddSRGBmZi3XDk0dgJmZNS0XBGZmLZwLAjOzFs4FgZlZC+eCwMyshduxqQOor86dO0f37t2bOgwzs2Zl7ty570REl3zbml1B0L17d+bMmdPUYZiZNSuSlm9tm5uGzMxaOBcEZmYtnAsCM7MWzgWBmVkL54LAzKyFK1pBIGmypLclLdjKdkm6TtISSfMlHVSsWMzMbOuKWSOYAhxZx/avAr3Sn9OB/y5iLGZmthVFe44gIp6Q1L2OJMcAt0cyDvYzkjpJ2isi3ipWTCveXc/f3l1fK84t0wVbrqydLt/g3fmG9N5izbaeL+9+hWPInybPym3Oc74jRYY0+Y5VeEj0bX6/GvH6ZXkPs5+v8Y5VO2Hm823rfgWOszWN+zdT93G2dqwt02TbrzFj2PKzoXCgh/fZkwH7dCqYrr6a8oGyvYEVOcuV6botCgJJp5PUGujWrds2nWzu8ncZ9Ztn+Giz518ws48nqe7te+zaZrsrCDKLiEnAJIDy8vJt+iT/27vr+Whz8LNj92ffLu1rbMt38fO9H1um2zJVlmMpT6Is51PG82WLKWPs2xjDlnnOG1mG8+WL6WNw/TLE0KDrXnBFtmNt6/XLGnvtVdn/l4p7/bKkqX2soseeJdAm0pQFwUpgn5zlrum6oqiqdX1u393p3nmXYp3GzKzZacrbR2cB307vHvocsKaY/QNVBcHHuFA2M2sSRasRSJoGDAU6S6oELgdaA0TETcADwHBgCbAe+E6xYoF/dczkrd6ambVgxbxraEyB7QGcVazz5zkf4BqBmVltLebJYt8rZGaWX4spCHAfgZlZXi2nIEh9nG/hMjNrCi2uIDAzs5pcEJiZtXAtpiDIN/6HmZllvH1U0g7AAOCTwAfAgoh4u5iBFYt7CMzMaqqzIJD0aeAHwJeA14BVQBvgM5LWA78BbouIzcUO1MzMiqNQjeDHJPME/EfUGqdV0h7ACcBJwG3FCc/MzIqtzoKgrqeD06ahXzV2QMWScah0M7MWZ5s7iyV9uTEDKRU/RmBmVlND7hq6pdGiMDOzJlOos3jW1jYBuzd+OGZmVmqFOosPBU4E1tVaL+DgokRkZmYlVaggeAZYHxGP194g6ZXihFQc7is2M8uv0F1DX61j22GNH07xeWIaM7OaWswQE2Zmlp8LAjOzFq7FFAR+oMzMLL8WUxBU8QNlZmY1ZS4IJF1R17KZmTVP9akRzC2wbGZmzVDmgiAi7q1r2czMmqdCQ0xMpI5nsSLinEaPqEg8Q5mZWX6FniyeU5IoSsh9xWZmNRV6srjGhDOS2kXE+uKGZGZmpZSpj0DSYEmLgJfT5QGSbixqZGZmVhJZO4t/BRwBrAaIiBeBZjXWkB8oMzPLrz53Da2oteqjRo6lNNxJYGZWQ6HO4iorJB0ChKTWwLnA4uKFZWZmpZK1RvBd4Cxgb+BNoCxdNjOzZi5TQRAR70TEtyJiz4joEhEnRsTqQvtJOlLSK5KWSLooz/Zukh6VNE/SfEnDtyUTWbiLwMwsv6x3De0r6V5JqyS9LelPkvYtsE8r4Abgq0BfYIykvrWSXQrcFREHAscDRb8TyRPTmJnVlLVp6E7gLmAv4JPATGBagX0OBpZExNKI2AhMB46plSaAXdPXHUmanczMrISyFgTtIuKOiPgw/fkd0KbAPnsDuXcaVabrcl0BnCipEngA+F6+A0k6XdIcSXNWrVqVMWQzM8uizoJA0m6SdgMelHSRpO6SPiXpQpIP7oYaA0yJiK7AcOAOSVvEFBGTIqI8Isq7dOnSCKc1M7MqhW4fnUvSfFPVsP4fOdsCuLiOfVcC++Qsd03X5ToVOBIgImZLagN0Bt4uEFf9+YkyM7O8Co011KMBx34e6CWpB0kBcDxwQq00fwMOB6ZI6kPS3FTUth/PUGZmVlPWB8qQ1J/k7p/qvoGIuH1r6SPiQ0lnAw8DrYDJEbFQ0nhgTkTMAs4Hbpb0fZIaxtgIf3U3MyulTAWBpMuBoSQFwQMkt4T+FdhqQQAQEQ9Qqy8hIi7Leb0IGFKviM3MrFFlvWvoOJImnL9HxHeAASS3ezYbrmaYmeWXtSD4ICI2Ax9K2pWkM3efAvt8LLmLwMyspqx9BHMkdQJuJrmTaB0wu1hBmZlZ6WQqCCLizPTlTZIeAnaNiPnFC8vMzEql0OT1B9W1LSJeaPyQisP3IpmZ5VeoRvCLOrYF8MVGjKUk5AcJzMxqKPRA2bBSBWJmZk0j81SVZma2fXJBYGbWwrWYgsAjV5iZ5Zd1hjJJOlHSZelyN0kHFze04nBXsZlZTVlrBDcCg0nmDwBYSzINpZmZNXNZnyweFBEHSZoHEBHvSdqpiHGZmVmJZK0RbEonow8ASV2AzUWLqgjcQ2Bmll/WguA64B5gD0k/IRmC+qqiRVVEfp7MzKymrGMNTZU0l2QoagFfi4jFRY3MzMxKIuvENNcB0yPCHcRmZtuZrE1Dc4FLJb0u6VpJ5cUMyszMSidTQRARt0XEcOCzwCvA1ZJeK2pkjczPk5mZ5VffJ4t7Ar2BTwEvN344xSc/UmZmVkPWJ4t/ntYAxgMLgPKIGFHUyMzMrCSyPlD2OjA4It4pZjBmZlZ6hWYo6x0RLwPPA90kdcvd3qxmKGvqAMzMPqYK1QjOA04n/0xlzXKGMncRmJnVVGiGstPTl1+NiA252yS1KVpUZmZWMlnvGno64zozM2tmCvUR/BuwN9BW0oH8q2FlV6BdkWNrVJ6Yxswsv0J9BEcAY4GuwISc9WuBHxYppqLyoHNmZjUV6iO4DbhN0jci4g8lisnMzEqoUNPQiRHxO6C7pPNqb4+ICXl2MzOzZqRQZ/Eu6e/2QIc8P3WSdKSkVyQtkXTRVtKMkrRI0kJJd9YjdjMzawSFmoZ+k/7+UX0PnM5odgPwZaASeF7SrIhYlJOmF3AxMCSd/nKP+p7HzMwapj5jDe0qqbWkv0haJenEArsdDCyJiKURsRGYDhxTK82/AzdExHsAEfF2fTNgZmYNk/U5gq9ExP8CRwPLSEYhHVdgn72BFTnLlem6XJ8BPiPpKUnPSDoy34EknS5pjqQ5q1atyhiymZllkbUgqGpCOgqYGRFrGun8OwK9gKHAGOBmSZ1qJ4qISRFRHhHlXbp0aaRTm5kZZC8I7pP0MjAQ+IukLsCGAvusBPbJWe6arstVCcyKiE0R8QbwKknBYGZmJZJ1hrKLgENI5iHYBPwfW7b31/Y80EtSD0k7AccDs2ql+SNJbQBJnUmaipZmDd7MzBou6+T1rYETgcOUPJr7OHBTXftExIeSzgYeBloBkyNioaTxwJyImJVu+4qkRcBHwLiIWL3NuTEzs3rLOjHNfwOtgRvT5ZPSdafVtVNEPAA8UGvdZTmvg2So6y0eVjMzs9LIWhB8NiIG5Cw/IunFYgRUbB5qyMyspqydxR9J+nTVgqR9SZpyzMysmctaIxgHPCppKcmX6k8B3ylaVGZmVjIFC4L0VtE1JE8KVw0B8UpE/LOYgZmZWWnU2TQk6TRgITARqAC6R8R8FwJmZtuPQjWC/wT6RcSqtF9gKls+C2BmZs1Yoc7ijRGxCiAilgI7Fz8kMzMrpUI1gq6SrtvackScU5ywzMysVAoVBLVHGJ1brEDMzKxpZJmz2MzMtmOF7hq6WVL/rWzbRdIpkr5VnNAaV0RTR2Bm9vFUqGnoBuAySfsDC4BVQBuSoaJ3BSaT3EnUbKSD5pmZWapQ01AFMEpSe6Ac2Av4AFgcEa8UPzwzMyu2TENMRMQ64LHihmJmZk0h66BzZma2nXJBYGbWwtWrIJDUrliBmJlZ08hUEEg6JJ1O8uV0eYCkGwvsZmZmzUDWGsEvgSOA1QAR8SJwWLGCMjOz0sncNBQRK2qt8gxlZmbbgawzlK2QdAgQkloD5wKLixeWmZmVStYawXeBs4C9gZVAGXBmkWIqisBjTJiZ5ZO1RrBfRNQYU0jSEOCpxg+puDzAhJlZTVlrBBMzrjMzs2amzhqBpMHAIUAXSeflbNoVaFXMwMzMrDQKNQ3tBLRP03XIWf+/wHHFCsrMzEqn0OijjwOPS5oSEctLFJOZmZVQ1s7i9ZKuAfqRzEcAQER8sShRmZlZyWTtLJ5KMrxED+BHwDLg+SLFZGZmJZS1INg9Im4BNkXE4xFxCuDagJnZdiBr09Cm9Pdbko4C3gR2K05IZmZWSllrBD+W1BE4H7gA+C3wn4V2knSkpFckLZF0UR3pviEpJJVnjMfMzBpJ1qkq70tfrgGGQfWTxVslqRVwA/BloBJ4XtKsiFhUK10HkrGLnq1f6PUTHmHCzCyvOmsEklpJGiPpAkn903VHS3oauL7AsQ8GlkTE0ojYCEwHjsmT7krgamBD/cOvP3mMCTOzGgo1Dd0CnAbsDlwn6XfAtcDPI+LAAvvuDeQOXV2Zrqsm6SBgn4i4v64DSTpd0hxJc1atWlXgtGZmVh+FmobKgQMiYrOkNsDfgU9HxOqGnljSDsAEYGyhtBExCZgEUF5e7kYeM7NGVKhGsDEiNgNExAZgaT0KgZXAPjnLXdN1VToA/YHHJC0DPgfMcoexmVlpFaoR9JY0P30t4NPpsoCIiAPq2Pd5oJekHiQFwPHACVUbI2IN0LlqWdJjwAURMafeuTAzs21WqCDos60HjogPJZ0NPEwyUunkiFgoaTwwJyJmbeuxzcys8RQadK5BA81FxAPAA7XWXbaVtEMbci4zM9s2mSevNzOz7ZMLAjOzFi5zQSCpraT9ihmMmZmVXqaCQNIIoAJ4KF0uk+TOXjOz7UDWGsEVJENGvA8QERUkcxOYmVkzl7Ug2JTe95/LT/iamW0Hss5HsFDSCUArSb2Ac4CnixeWmZmVStYawfdI5iv+J3AnyXDU/1mkmMzMrISy1gh6R8QlwCXFDMbMzEova43gF5IWS7qyal4CMzPbPmQqCCJiGMnMZKuA30h6SdKlRY3MzMxKIvMDZRHx94i4DvguyTMFeccMMjOz5iXrA2V9JF0h6SVgIskdQ12LGpmZmZVE1s7iycAM4IiIeLOI8ZiZWYllKggiYnCxAzEzs6ZRZ0Eg6a6IGJU2CeU+SZxlhjIzM2sGCtUIzk1/H13sQMzMrGnU2VkcEW+lL8+MiOW5P8CZxQ/PzMyKLevto1/Os+6rjRmImZk1jUJ9BGeQfPPfV9L8nE0dgKeKGZiZmZVGoT6CO4EHgZ8CF+WsXxsR7xYtKjMzK5lCBUFExDJJZ9XeIGk3FwZmZs1flhrB0cBckttHlbMtgH2LFJeZmZVInQVBRByd/va0lGZm26msYw0NkbRL+vpESRMkdStuaGZmVgpZbx/9b2C9pAHA+cDrwB1Fi8rMzEoma0HwYUQEcAxwfUTcQHILqZmZNXNZRx9dK+li4CTgUEk7AK2LF5aZmZVK1hrBaJKJ60+JiL+TzEVwTdGiMjOzksk6VeXfgalAR0lHAxsi4vaiRmZmZiWR9a6hUcBzwDeBUcCzko7LsN+Rkl6RtETSRXm2nydpkaT5kv4i6VP1zYCZmTVM1j6CS4DPRsTbAJK6AH8Gfr+1HSS1Am4gGbCuEnhe0qyIWJSTbB5QHhHr03GNfk7SDGVmZiWStY9gh6pCILU6w74HA0siYmlEbASmk9x1VC0iHo2I9eniM3geZDOzkstaI3hI0sPAtHR5NPBAgX32BlbkLFcCg+pIfyrJAHdbkHQ6cDpAt25+js3MrDFlnbN4nKRjgc+nqyZFxD2NFYSkE4Fy4AtbOf8kYBJAeXl55EtjZmbbptB8BL2Aa4FPAy8BF0TEyozHXgnsk7PcNV1X+xxfIumD+EJE/DPjsc3MrJEUauefDNwHfINkBNKJ9Tj280AvST0k7QQcD8zKTSDpQOA3wMhafRBmZlYihZqGOkTEzenrVyS9kPXAEfGhpLOBh4FWwOSIWChpPDAnImaRPJTWHpgpCeBvETGy3rkwM7NtVqggaJN+a6+ah6Bt7nJE1FkwRMQD1OpUjojLcl5/qd4Rm5lZoypUELwFTMhZ/nvOcgBfLEZQZmZWOoUmphlWqkDMzKxpZH2gzMzMtlMuCMzMWjgXBGZmLVzW0UeVzlV8WbrcTdLBxQ3NzMxKIWuN4EZgMDAmXV5LMrKomZk1c1kHnRsUEQdJmgcQEe+lTwubmVkzl7VGsCmdXyCgej6CzUWLyszMSiZrQXAdcA+wh6SfAH8FripaVGZmVjJZh6GeKmkucDjJ8BJfi4jFRY3MzMxKIlNBIKkbsB64N3ddRPytWIGZmVlpZO0svp+kf0BAG6AH8ArQr0hxmZlZiWRtGto/d1nSQcCZRYnIzMxKapueLE6Hn65r/mEzM2smsvYRnJezuANwEPBmUSIyM7OSytpH0CHn9YckfQZ/aPxwzMys1AoWBOmDZB0i4oISxGNmZiVWZx+BpB0j4iNgSIniMTOzEitUI3iOpD+gQtIsYCbwf1UbI+LuIsZmZmYlkLWPoA2wmmSO4qrnCQJwQWBm1swVKgj2SO8YWsC/CoAqUbSobLuxadMmKisr2bBhQ1OHYtYitGnThq5du9K6devM+xQqCFoB7alZAFRxQWAFVVZW0qFDB7p3746U78/IzBpLRLB69WoqKyvp0aNH5v0KFQRvRcT4hoVmLdmGDRtcCJiViCR23313Vq1aVa/9Cj1Z7P9eazAXAmalsy3/b4UKgsO3LRQzM2su6iwIIuLdUgViViytWrWirKyM/v37M2LECN5///1GOe6UKVM4++yzG+VY3bt3Z//996esrIyysjKefvrpRjlubRUVFTzwwAM11j344IOUl5fTt29fDjzwQM4//3wArrjiCq699tpGO/chhxxS/XrcuHH069ePcePGcdNNN3H77bc36Njz5s3j1FNPrbHua1/7Gp/73OdqrBs7diy///3va6xr37599etXX32V4cOH06tXLw466CBGjRrFP/7xjwbFNnPmTPr168cOO+zAnDlztpruoYceYr/99qNnz5787Gc/q17/xhtvMGjQIHr27Mno0aPZuHEjANdffz2TJ09uUGxVtmnQObPmpG3btlRUVLBgwQJ22203brjhhqYOKa9HH32UiooKKioqanxo1uXDDz+s1zlqFwQLFizg7LPP5ne/+x2LFi1izpw59OzZs17HzCq3cJs0aRLz58/nmmuu4bvf/S7f/va3Mx8nX56vuuoqzjnnnOrl999/n7lz57JmzRqWLl2a6bgbNmzgqKOO4owzzuC1117jhRde4Mwzz6x3e3tt/fv35+677+awww7bapqPPvqIs846iwcffJBFixYxbdo0Fi1aBMAPfvADvv/977NkyRI+8YlPcMsttwBwyimnMHHixAbFViXrcwRmDfajexey6M3/bdRj9v3krlw+Ivu0GIMHD2b+/PkAPPfcc5x77rls2LCBtm3bcuutt7LffvsxZcoUZs2axfr163n99df5+te/zs9//nMAbr31Vn7605/SqVMnBgwYwM477wzAsmXLOOWUU3jnnXfo0qULt956K926dWPs2LG0bduWefPm8fbbbzN58mRuv/12Zs+ezaBBg5gyZcpWY63rmG3atGHevHkMGTKEs846i7POOotVq1bRrl07br75Znr37s3MmTP50Y9+RKtWrejYsSN//vOfueyyy/jggw/461//ysUXX8z999/PJZdcQu/evYGk9nTGGWdsEcvNN9/MpEmT2LhxIz179uSOO+6gXbt2W5zjiSeeYOHChXznO99h48aNbN68mT/84Q/06tWL9u3bs27dOkaOHMm6desYOHAgF198MYsXL6Z9+/ZccMEFvP7663nzUjvPEyZMqI5t7dq1zJ8/nwEDBlSvu/vuuxkxYgR77rkn06dP54c//GHBv40777yTwYMHM2LEiOp1Q4cOLbhfIX369CmY5rnnnqNnz57su+++ABx//PH86U9/ok+fPjzyyCPceeedAJx88slcccUVnHHGGbRr147u3bvz3HPPcfDBBzcoRtcIrMX46KOP+Mtf/sLIkSMB6N27N08++STz5s1j/PjxNT4sKioqmDFjBi+99BIzZsxgxYoVvPXWW1x++eU89dRT/PWvf63+xgbwve99j5NPPpn58+fzrW99q8a30/fee4/Zs2fzy1/+kpEjR/L973+fhQsX8tJLL1FRUVGdbtiwYZSVlTFo0KCCx6ysrOTpp59mwoQJnH766UycOJG5c+dy7bXXcuaZyVQh48eP5+GHH+bFF19k1qxZ7LTTTowfP57Ro0dTUVHB6NGjWbBgAQMHDix47Y499lief/55XnzxRfr06VP9rbT2OQBuuukmzj33XCoqKpgzZw5du3atcaxZs2ZV19JGjx5dY9vW8lI7z7nmzJlD//79a6ybNm0aY8aMYcyYMUybNq1g/oDM12Lt2rXVTXi1f3L/Jupj5cqV7LPPPtXLXbt2ZeXKlaxevZpOnTqx44471lhfpby8nCeffHKbzpnLNQIrmfp8c29MH3zwAWVlZaxcuZI+ffrw5S9/GYA1a9Zw8skn89prryGJTZs2Ve9z+OGH07FjRwD69u3L8uXLeeeddxg6dChdunQBYPTo0bz66qsAzJ49m7vvTh60P+mkk7jwwgurjzVixAgksf/++7Pnnnuy//7JPE/9+vVj2bJllJWVAUnTUOfOnav3q+uY3/zmN2nVqhXr1q3j6aef5pvf/Gb1tn/+858ADBkyhLFjxzJq1CiOPfbYBl3DBQsWcOmll/L++++zbt06jjjiiK2eY/DgwfzkJz+hsrKSY489ll69emU6R115yc1zbW+99Vb1ewLwj3/8g9dee43Pf/7zSKJ169YsWLCA/v37572jpr532XTo0KFGAd6U9thjD15++eUGH6eoNQJJR0p6RdISSRfl2b6zpBnp9mcldS9mPNYyVX37XL58ORFR3UfwX//1XwwbNowFCxZw77331nj6uarJB5Lmkvq2xeeqOtYOO+xQ47g77LDDNh93l112AWDz5s106tSpum+hoqKCxYsXA8k38x//+MesWLGCgQMHsnr16i2O069fP+bOnVvwfGPHjuX666/npZde4vLLL6++VvnOccIJJ1R/6x8+fDiPPPJIpjzVlZfcPNfWtm3bGu/dXXfdxXvvvUePHj3o3r07y5Ytq64V7L777rz33nvVad99993qwjfrtShGjWDvvfdmxYoV1cuVlZXsvffe7L777rz//vvVfydV66tUNWs2VNEKgnT46huArwJ9gTGS+tZKdirwXkT0BH4JXF2seMzatWvHddddxy9+8Qs+/PBD1qxZU/1PVVdbfZVBgwbx+OOPs3r1ajZt2sTMmTOrtx1yyCFMnz4dgKlTp3LooYc2ON4sx9x1113p0aNHdSwRwYsvvgjA66+/zqBBgxg/fjxdunRhxYoVdOjQgbVr11bvP27cOK666qrqms3mzZu56aabtjjP2rVr2Wuvvdi0aRNTp06tXp/vHEuXLmXfffflnHPO4ZhjjqnukymkrrzUpU+fPixZsqR6edq0aTz00EMsW7aMZcuWMXfu3OrrOHToUGbMmFF9582UKVMYNmwYACeccAJPP/00999/f/WxnnjiCRYsWFDjfFU1gnw/ffvW/ojL5rOf/SyvvfYab7zxBhs3bmT69OmMHDkSSQwbNqz6TqfbbruNY445pnq/V199dYtmsW1RzBrBwcCSiFgaERuB6cAxtdIcA9yWvv49cLj89JEV0YEHHsgBBxzAtGnTuPDCC7n44os58MADM30z32uvvbjiiisYPHgwQ4YMqdEJOHHiRG699VYOOOAA7rjjDn796183ONasx5w6dSq33HILAwYMoF+/fvzpT38Ckg/5/fffn/79+3PIIYcwYMAAhg0bxqJFiygrK2PGjBkccMAB/OpXv2LMmDH06dOH/v37573L5sorr2TQoEEMGTKkumN5a+e466676N+/P2VlZSxYsKBedwRtLS916d27N2vWrGHt2rUsW7aM5cuX17httEePHnTs2JFnn32Wo48+mkMPPZSBAwdSVlbGU089xdVXJ98/27Zty3333cfEiRPp1asXffv25cYbb6zR7LQt7rnnHrp27crs2bM56qijqpvV3nzzTYYPHw7AjjvuyPXXX88RRxxBnz59GDVqFP36JU2pV199NRMmTKBnz56sXr26xm2yTz31VHVTZ0MoojhDBkk6DjgyIk5Ll08CBkXE2TlpFqRpKtPl19M079Q61unA6QDdunUbuHz58nrH8z8L/84fK1YyYVQZbVpv2c5oxbF48eJMd02YNcQvf/lLOnTowGmnndbUoZTMvHnzmDBhAnfccccW2/L930maGxHl+Y7VLO4aiohJEVEeEeXbWjp/pd+/ceO3BroQMNsOnXHGGTX6X1qCd955hyuvvLJRjlXMu4ZWAvvkLHdN1+VLUylpR6AjybwHZmaZtWnThpNOOqmpwyipxmgSqlLMGsHzQC9JPSTtBBwPzKqVZhZwcvr6OOCRKFZblTUZv6VmpbMt/29FKwgi4kPgbOBhYDFwV0QslDRe0sg02S3A7pKWAOcBW9xias1bmzZtWL16tQsDsxKomo+gTZs29dqvaJ3FxVJeXh51DdxkHy+eocystLY2Q1ldncV+stiKqnXr1vWaKcnMSq9Z3DVkZmbF44LAzKyFc0FgZtbCNbvOYkmrgPo/WpzoDLxTMNX2xXluGZznlqEhef5UROR9IrfZFQQNIWnO1nrNt1fOc8vgPLcMxcqzm4bMzFo4FwRmZi1cSysIJjV1AE3AeW4ZnOeWoSh5blF9BGZmtqWWViMwM7NaXBCYmbVw22VBIOlISa9IWiJpixFNJe0saUa6/VlJ3ZsgzEaVIc/nSVokab6kv0j6VFPE2ZgK5Tkn3TckhaRmf6thljxLGpW+1wsl3VnqGBtbhr/tbpIelTQv/fse3hRxNhZJkyW9nc7gmG+7JF2XXo/5kg5q8EkjYrv6AVoBrwP7AjsBLwJ9a6U5E7gpfX08MKOp4y5BnocB7dLXZ7SEPKfpOgBPAM8A5U0ddwne517APOAT6fIeTR13CfI8CTgjfd0XWNbUcTcwz4cBBwELtrJ9OPAgIOBzwLMNPef2WCM4GFgSEUsjYiMwHTimVppjgNvS178HDpekEsbY2ArmOSIejYj16eIzJDPGNWdZ3meAK4Grge1hHOwsef534IaIeA8gIt4ucYyNLUueA9g1fd0ReLOE8TW6iHgCeLeOJMcAt0fiGaCTpL0acs7tsSDYG1iRs1yZrsubJpIJdNYAu5ckuuLIkudcp5J8o2jOCuY5rTLvExH3lzKwIsryPn8G+IykpyQ9I+nIkkVXHFnyfAVwoqRK4AHge6UJrcnU9/+9IM9H0MJIOhEoB77Q1LEUk6QdgAnA2CYOpdR2JGkeGkpS63tC0v4R8X5TBlVkY4ApEfELSYOBOyT1j4jNTR1Yc7E91ghWAvvkLHdN1+VNI2lHkurk6pJEVxxZ8oykLwGXACMj4p8liq1YCuW5A9AfeEzSMpK21FnNvMM4y/tcCcyKiE0R8QbwKknB0FxlyfOpwF0AETEbaEMyONv2KtP/e31sjwXB80AvST0k7UTSGTyrVppZwMnp6+OARyLthWmmCuZZ0oHAb0gKgebebgwF8hwRayKic0R0j4juJP0iIyOiOc9zmuVv+48ktQEkdSZpKlpawhgbW5Y8/w04HEBSH5KCYFVJoyytWcC307uHPgesiYi3GnLA7a5pKCI+lHQ28DDJHQeTI2KhpPHAnIiYBdxCUn1cQtIpc3zTRdxwGfN8DdAemJn2i/8tIkY2WdANlDHP25WMeX4Y+IqkRcBHwLiIaLa13Yx5Ph+4WdL3STqOxzbnL3aSppEU5p3Tfo/LgdYAEXETST/IcGAJsB74ToPP2Yyvl5mZNYLtsWnIzMzqwQWBmVkL54LAzKyFc0FgZtbCuSAwM2vhXBC0AJI+klSR89O9jrTrGuF8UyS9kZ7rhfRpz/oe47eS+qavf1hr29MNjTE9TtV1WSDpXkmdCqQv25aRLSXtJem+9PVQSWvS8y6WdPk2HG9k1Sickr5WdZ3S5fHpg4MNkr6HxxVI81h9HtBL835fhnR5R9+UdK2kL2Y9n2XngqBl+CAiynJ+lpXgnOMiogy4iORBtnqJiNMiYlG6+MNa2w5peHjAv65Lf5LnSc4qkL6M5P7t+joPuDln+cn02pSTjJFTr2GEI2JWRPwsXfwayYibVdsui4g/b0OMHydTgHxjJE0k+XuyRuaCoAWS1F7JnAQvSHpJ0hajdqbfYp/I+cZ8aLr+K5Jmp/vOlNS+wOmeAHqm+56XHmuBpP9M1+0i6X5JL6brR6frH5NULulnQNs0jqnptnXp7+mSjsqJeYqk4yS1knSNpOeVjNf+Hxkuy2zSgbskHZzmcZ6kpyXtlz7VOh4YncYyOo19sqTn0rT5Rj8F+AbwUO2VEfF/wFygZ1rbeCaN9x5Jn0hjOUf/mkdierpurKTrJR0CjASuSWP6dM41OFLSzJxrU/1tvL7voaTL0mu5QNIkqcZIvSfl/I0cnKbPel3y2tromxGxHNhd0r/V53iWQVOMt+2f0v6QPGFakf7cQ/JE+a7pts4kTyhWPVy4Lv19PnBJ+roVydg9nUk+2HdJ1/8AuCzP+aYAx6Wvvwk8CwwEXgJ2IXnCeSFwIMmH5M05+3ZMfz9GOn9AVUw5aapi/DpwW/p6J5IRGdsCpwOXput3BuYAPfLEuS4nfzOBI9PlXYEd09dfAv6Qvh4LXJ+z/1XAienrTiTj+uxS6xw9gLk5y0OB+9LXuwPLgH7AfOAL6frxwK/S128CO1edo3Ycudc6dzl9j/+W8179N3DiNr6Hu+WsvwMYkfMe3Zy+Pox0/PytXZdaeS8HflvH32x38ozHT1Kz+kZT/09tbz/b3RATltcHkTRFACCpNXCVpMOAzSTfhPcE/p6zz/PA5DTtHyOiQtIXSJohnkq/FO5E8k06n2skXUoy5supJGPB3BPJt2Ak3Q0cSvJN+ReSrib5kHiyHvl6EPi1pJ1JmhKeiIgPJH0FOCCnjbsjycBrb9Tav62kijT/i4H/l5P+Nkm9SIYsaL2V838FGCnpgnS5DdAtPVaVvdhy3JtDJc0jufY/IxkorlNEPJ5uv42kYIKkgJgq6Y8k4whlEsnQDA8BIyT9HjgKuJBk1Nms72GVYZIuBNoBu5EU4vem26al53tC0q5K+lm2dl1y45sDnJY1PzneBj65DftZHVwQtEzfAroAAyNik5LROdvkJkj/sQ8j+QCZImkC8B7w/yJiTIZzjIuI31ctSDo8X6KIeDVtIx8O/FjSXyJifJZMRMQGSY8BRwCjSSYtgWTmpu9FxMMFDvFBRJRJakcyls1ZwHUkk9k8GhFfV9Kx/thW9hfJt9NX6joHta4tSR/B0dUHkTrWsf9RJN+2RwCXSNq/jrS1TQfOJmlmmRMRa9NmnazvIZLaADeS1M5WSLqCmvmpPUZNsJXrImnPesS+NW1Irqk1IvcRtEwdgbfTQmAYsMX8xUrmNP5HRNwM/JZk6rxngCGSqtr8d5H0mYznfBL4mqR2knYhadZ5UtIngfUR8TuSgfHydZxuSmsm+cwgGXSrqnYByYf6GVX7SPpMes68Ipm57RzgfP1rWPKqYX3H5iRdS9JEVuVh4HtVbeZKRnit7VWSZo6tiog1wHtK+2GAk4DHlcypsE9EPErShNORpFktV+2Ycj1Ocj3/nX8VkvV9D6s+9N9J+xJq30lU1afzeZJRMNeQ7bpsq88AeefytW3ngqBlmgqUS3oJ+Dbwcp40Q4EX0yaM0cCvI2IVyQfjNEnzSZoUemc5YUS8QNLu/BxJn8FvI2IesD/wXNpEcznw4zy7TwLmK+0sruV/SJo7/hzJVIaQFFyLgBeU3IL4GwrUftNY5pNMcvJz4Kdp3nP3exToW9VZTFJzaJ3GtjBdrn3c/wNer/rgrcPJJM1p80nuThpP0nfxu/R9mgdcF1tOMDMdGJd2yn661rk/Au4Dvpr+pr7vYXq+m0k+fB8maTLMtSG9TjeRNAFChuui5EaA3+Y7p5LRN2cD+0mqlHRqur41yY0HzXko8Y8ljz5qVmSSvk7SDHdpU8fSnKXX8aCI+K+mjmV74z4CsyKLiHskNec5sT8udgR+0dRBbI9cIzAza+HcR2Bm1sK5IDAza+FcEJiZtXAuCMzMWjgXBGZmLdz/B1NqPYbpMFemAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Training results\n",
        "y_train_rfc_pred = rfc.predict(X_train_pred)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_train, y_train_rfc_pred)))\n",
        "print('Precision Score:', (precision_score(y_train, y_train_rfc_pred)))\n",
        "print('Recall Score:', (recall_score(y_train,y_train_rfc_pred)))\n",
        "print('F1 Score:', (f1_score(y_train, y_train_rfc_pred)))\n",
        "\n",
        "y_score_rf = rfc.predict_proba(X_train_pred)\n",
        "print('ROC Score:', (roc_auc_score(y_train, y_score_rf[:,1])))\n",
        "print(plot_roc_curve(rfc, X_train_pred, y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 123,
      "metadata": {
        "id": "cGgamL8wtrsV"
      },
      "outputs": [],
      "source": [
        "#Finding X_test_pred from making mnb predictions\n",
        "nb_title_test_pred = nb_title_tuned.predict(X_test_title_bow)\n",
        "nb_profile_test_pred = nb_profile_tuned.predict(X_test_profile_bow)\n",
        "nb_description_test_pred = nb_description_tuned.predict(X_test_description_bow)\n",
        "nb_requirements_test_pred = nb_requirements_tuned.predict(X_test_requirements_bow)\n",
        "nb_benefits_test_pred = nb_benefits_tuned.predict(X_test_benefits_bow)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 124,
      "metadata": {
        "id": "p9p_7h6kuEnu"
      },
      "outputs": [],
      "source": [
        "X_test_pred = [nb_title_test_pred, nb_profile_test_pred, nb_description_test_pred, nb_description_test_pred, nb_benefits_test_pred]\n",
        "X_test_pred = np.array(X_test_pred)\n",
        "X_test_pred = X_test_pred.T"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iZFZVZQO2ejb"
      },
      "source": [
        "I fitted the BOW models into a random forest and generated a f1 of 0.8."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 125,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "k9e_lL0Lu0QF",
        "outputId": "cee5d1dc-2d96-4e60-fdb1-a113c706d081"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9871364653243848\n",
            "Precision Score: 0.98\n",
            "Recall Score: 0.6901408450704225\n",
            "F1 Score: 0.8099173553719008\n",
            "ROC Score: 0.8817213121477848\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A0B5AC3040>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu9ElEQVR4nO3deXwV5dn/8c9FCCTsyCaCCGIsOwHyyFYtlFJX0ForxWql1Yen7k9V/Gn1UYutS91a90JV0FJAbW1TtbVqFS2gJUjEAFWRQgmiAgKCEAhw/f6YSTwnJDmT5ZyQ5Pt+vfLK7HPNOclcM/c9c9/m7oiISOPVpK4DEBGRuqVEICLSyCkRiIg0ckoEIiKNnBKBiEgj17SuA6iqjh07es+ePes6DBGRemXp0qWb3b1TefPqXSLo2bMneXl5dR2GiEi9YmbrKpqnoiERkUZOiUBEpJFTIhARaeSUCEREGjklAhGRRi5picDMHjOzT82soIL5Zmb3mdlqM1tuZkOTFYuIiFQsmXcEs4CTKpl/MpAV/kwFHk5iLCIiUoGkJQJ3fx34rJJFTgee8MCbQDsz65qseERE6qv9B5yfP7+SDdt2J2X7dVlH0A1YHzNeGE47iJlNNbM8M8vbtGlTSoITETkUHDjgXPPMcma+8W9ee+/TpOyjXlQWu/sMd89x95xOncp9Q1pEpMFxd27KXcHv3y7kx984lu8NPyop+6nLRLABODJmvHs4TUSk0XN3bv/Lv3jyzXX8zwlHc/m4Y5K2r7pMBLnA98Onh0YA2919Yx3GIyJyyLjvldX8+vU1nDfiKK49uQ9mlrR9Ja3ROTObC4wBOppZIXATkA7g7o8ALwCnAKuBXcAPkhWLiEh9MvP1Ndz78vucNaw7P53YP6lJAJKYCNx9coL5DlySrP2LiByq9h9wPv68iA1bd7Nh267w924Kw99rNn3BqYO6cse3B9GkSXKTANTDZqhFRA51e/bt56Nt8Sf6wm27S0/4G7cXsf+Ax63TsVUzurXLpM/hrTlzSDemntCbtBQkAVAiEBGpsp179pWe5Au37j7oRL9px5645ZsYHN4mg27tM/mvnofRrV0m3dpnxv3OSE+ro6NRIhARiePufPbFXjbEnNhLimxKxrfvLo5bp1nTJsFJvV0mX/9K54NO8oe3zSA97dB9Wl+JQEQalf0HnE93FB10kg+u7Hfx0bYidhfvj1unVfOmpSf2YUe1Lz3Bd28fTOvYsnlKyvKTRYlARBqUPfv2s3FbUekV/JdFNruC8vltRewrUz7foWUzurXP5NgurRlb5oq+e7sWtMlsmvQnd+qSEoGI1Ctf7NlXzkk+uJrfsG03n+7Yg8ec55sYdGmTQbd2mQzt0Z5ugzLjruiPaJdJi2aN+1QY6ejNrAkwGDgC2A0UuHtyGr0QkUbL3dm2q7i0qKYwPLmXnuy37WbbrjLl82lN6NouONGfkNWJ7u1bxJ3oD/Xy+UNBpYnAzHoD/w/4BvABsAnIAI41s13Ar4HZ7n4g2YGKSP134IDz6Y49Xz5tE3dFH/zetTe+fL5ls7TSE/uQHu3o1i7+RN+pVf0unz8UJLoj+BlBPwH/E74AVsrMOgPnAOcBs5MTnojUJ3v3HeDj7UUUxrwkFVspu3H7bor3x5fPt2+RTrf2mRzdqSXHZ3WKO8l3b59J28z0Bl0+fyioNBFU9nZwWDT0y9oOSEQOXbv27iunbP7L35/sKIornzeDzq2b061dJoOPbMepg7rGVMIG5fMtmzfu8vlDQbW/ATMb7+4v1WYwIlJ33J3tu4srLLLZsG03n32xN26d9DSja9vgCv6rWR3jTvLd2mfStW0mzZqqfP5QV5NU/CjQo7YCEZHkOnDA2bxzT9zVfOHW+CKcL8qUz2emf1k+P7B727gim27tWtCpdfOUNYMgyZOosji3ollAh9oPR0Sqq3h/WD4fdyX/5VM3H20rYu/++Oc62mam0719Jj07tGT0MR2/fEkqrJBt30Ll841BojuC44FzgZ1lphtwXFIiEpFy7d67v7SIpmyrlRu27ubjz4so855UUD7fPpMB3dpy4oDDS4tsSk70rVQ+LyROBG8Cu9x9QdkZZvZeckISaZy27y4OKmLLeXZ+w9bdbClTPt+0iXF42+D5+RG9O9C9fYuYE30mXdtl0Lxp3TVkJvVHoqeGTq5k3gm1H45Iw+TubNq5p9wnbUp+79izL26djPSwIbP2Leh/RNuwyObLE32XNhkqn5daoftCkVqwb/+BmI5G4q/mS8rs9+6LL59vk9GUbu1b0L19C0Yc3eGgpok7tGym8nlJCSUCkQiKivfzUTnNEZf8/vjz8joaCcrn+3Vtwzf7dTmoaeLWGel1dDQi8ZQIRIDPi4Ly+bLl8iWPWm7eGd/RSFoTK+1oZHivww46yR9Rxx2NiFSFEoE0eO7Oli/2xl3Fl1TIllzh7yiKL59vXtLRSPtM+vbtHHeS735YC7q0bk5TNWQmDUTkRGBmN7v7zRWNi9SVRB2Bf7RtN0XF8eXzrZs3Dd6Ajbui/7Ixs46tVD4vjUdV7giWJhgXSYqadgQ+rk/n0qdvSq7s22aqfF6kRORE4O5/rmxcpLpq0hF4Tmm3gS1Kuw2s647AReqbRE1M3A94RfPd/fJaj0galGp1BJ7WpPSEXh87AhepbxLdEeSlJAqpt2qzI/CSVis7qqMRkZRK9GZxXIczZtbC3XclNyQ5lFSnI/DDWgbl81mdG2dH4CL1TdQ+i0cSNDvdCuhhZoMJei27OJnBSfJVtSNwKymfV0fgIg1G1P/YXwInArkA7v6OmamtoUNcdToCT08zjmiXWdoReOzV/JHtW6h8XqQBqspTQ+vL3M7vr2hZSY3qdATeolla6YldHYGLCERPBOvNbBTgZpYOXAGsSl5YArXfEXi3dpm0U0cjIlJG1ETwI+BXQDfgI+BF4JJkBdVY1LQj8FMGdi190qakfF4dgYtIVUU6a7j7ZuB7Vd24mZ1EkEDSgN+4++1l5vcAZgPtwmWudfcXqrqfQ1F1OgJv2uTL8nl1BC4iqRL1qaGjCU7oIwheMFsM/Njd11SyThrwIDAeKASWmFmuu6+MWewG4Cl3f9jM+gEvAD2rcyCpVpsdgZec8Du3VkcjIpJ6UcsRfkdwUv9WOP5dYC4wvJJ1jgNWlyQLM5sHnA7EJgIH2oTDbQmKnQ4J1e0IvFu7TI7q0JJRvTvSvb06AheRQ1/URNDC3Z+MGf+tmU1LsE43YH3MeCEHJ46bgb+Z2WVAS+Ab5W3IzKYCUwF69OgRMeTKqSNwEZFAoraGDgsH/2Jm1wLzCK7iJxEU49TUZGCWu98dvrT2pJkNcPe4S213nwHMAMjJyamw7aNEFry/iXv+9h6FUToCL3OS79o2Qw2ZiUiDlOgSdinBib+kPON/YuY5cF0l624AjowZ7x5Oi3UBcBKAuy82swygI/Bpgriq5bl3PuL9T3ZyxpAj6B7TJLE6AheRxixRW0O9arDtJUCWmfUiSADfBc4ps8x/gHHALDPrC2QAm2qwz0pt+WIvR3dqyW1nDkrWLkRE6p2q9FA2AOhHcLIGwN2fqGh5d99nZpcSvHOQBjzm7ivMbDqQ5+65wFXATDP7McEdxhR3r3bRTyJbvthLh1bNk7V5EZF6KerjozcBYwgSwQvAycA/gAoTAUD4TsALZabdGDO8EhhdpYhrYMvOPfTu2DJVuxMRqReivp10FkERzsfu/gNgMMHjnvXKlp176dCqWV2HISJySImaCHaHT/LsM7M2BJW5RyZY55Cya+8+dhfv57CWKhoSEYkVtY4gz8zaATMJniTaSfB2cb2xZWfwuKjuCERE4kVta6ikA5pHzOyvQBt3X568sGrf5p1BB+gdlQhEROIkeqFsaGXz3P3t2g8pOT4v2gcEzUCIiMiXEt0R3F3JPAe+XouxJFXJU6lq60dEJF6iF8rGpioQERGpG2rcXkSkkVMiEBFp5JQIREQauUiJwALnmtmN4XgPMzsuuaGJiEgqRL0jeAgYSdB/AMAOgh7LRESknov6ZvFwdx9qZssA3H2rmenNLBGRBiDqHUFx2Bm9A5hZJ+BA5auIiEh9EDUR3Ac8C3Q2s58TNEF9a9KiEhGRlIna1tAcM1tK0BS1AWe4+6qkRiYiIikRtWOa+4B57q4KYhGRBiZq0dBS4AYz+9DM7jKznGQGJSIiqRMpEbj7bHc/Bfgv4D3gDjP7IKmR1bKkdYQsIlLPVfXN4mOAPsBRwL9qP5zkOXAgSAVpan1URCRO1DeLfxHeAUwHCoAcd5+Q1Mhq2f6SRNBEiUBEJFbUF8o+BEa6++ZkBpNMB8L+CJrojkBEJE6iHsr6uPu/gCVADzPrETu/PvVQFt4Q6I5ARKSMRHcEVwJTKb+nsnrVQ1lJ0ZDygIhIvEQ9lE0NB09296LYeWaWkbSokqC0aEiZQEQkTtSnhhZFnHbIKkkEempIRCReojqCw4FuQKaZDSFoXgKgDdAiybHVqv1hE3mqLBYRiZeojuBEYArQHbgnZvoO4CdJiikpSt4jaKI+2URE4iSqI5gNzDazb7v771MUU1KUFg2pjkBEJE6ioqFz3f23QE8zu7LsfHe/p5zVDkn79R6BiEi5EhWUtAx/twJal/NTKTM7yczeM7PVZnZtBcucbWYrzWyFmf2uCrFXSWnRkBKBiEicREVDvw5//7SqGw57NHsQGA8UAkvMLNfdV8YskwVcB4wOu7/sXNX9RKUXykREyleVtobamFm6mb1iZpvM7NwEqx0HrHb3Ne6+F5gHnF5mmf8GHnT3rQDu/mlVDyAqvVAmIlK+qM/QfNPdPwdOA9YStEI6LcE63YD1MeOF4bRYxwLHmtlCM3vTzE4qb0NmNtXM8swsb9OmTRFDjqcXykREyhc1EZQUIZ0KPO3u22tp/02BLGAMMBmYaWbtyi7k7jPcPcfdczp16lStHe1XM9QiIuWKmgieM7N/AcOAV8ysE1CUYJ0NwJEx493DabEKgVx3L3b3fwPvEySGpFEeEBGJF7WHsmuBUQT9EBQDX3BweX9ZS4AsM+tlZs2A7wK5ZZb5I8HdAGbWkaCoaE3U4EVEpOaidl6fDpwLnGDBJfUC4JHK1nH3fWZ2KfAikAY85u4rzGw6kOfuueG8b5rZSmA/MM3dt1T7aEREpMqidkzzMJAOPBSOnxdOu7Cyldz9BeCFMtNujBl2gqauD3pZTUREUiNqIvgvdx8cM/53M3snGQGJiEhqRa0s3m9mvUtGzOxogqIcERGp56LeEUwDXjWzNQRNUR8F/CBpUYmISMokTATho6LbCd4ULmkC4j1335PMwEREJDUqLRoyswuBFcD9QD7Q092XKwmIiDQcie4I/hfo7+6bwnqBORz8LoCIiNRjiSqL97r7JgB3XwM0T35IIiKSSonuCLqb2X0Vjbv75ckJS0REUiVRIijbwujSZAUiIiJ1I0qfxSIi0oAlempoppkNqGBeSzP7oZl9LzmhiYhIKiQqGnoQuNHMBgIFwCYgg6Cp6DbAYwRPEomISD2VqGgoHzjbzFoBOUBXYDewyt3fS354IiKSbJGamHD3ncBryQ1FRETqQtRG50REpIFSIhARaeSqlAjMrEWyAhERkboRKRGY2aiwO8l/heODzeyhBKuJiEg9EPWO4F7gRGALgLu/A5yQrKBERCR1IhcNufv6MpPUQ5mISAMQtYey9WY2CnAzSweuAFYlLywREUmVqHcEPwIuAboBG4Bs4OIkxSQiIikU9Y7gK+4e16aQmY0GFtZ+SCIikkpR7wjujzhNRETqmUrvCMxsJDAK6GRmV8bMagOkJTMwERFJjURFQ82AVuFyrWOmfw6claygREQkdRK1ProAWGBms9x9XYpiEhGRFIpaWbzLzO4E+hP0RwCAu389KVGJiEjKRK0snkPQvEQv4KfAWmBJkmISEZEUipoIOrj7o0Cxuy9w9x8CuhsQEWkAohYNFYe/N5rZqcBHwGHJCUlERFIp6h3Bz8ysLXAVcDXwG+B/E61kZieZ2XtmttrMrq1kuW+bmZtZTsR4RESklkTtqvK5cHA7MBZK3yyukJmlAQ8C44FCYImZ5br7yjLLtSZou+itqoUuIiK1odI7AjNLM7PJZna1mQ0Ip51mZouABxJs+zhgtbuvcfe9wDzg9HKWuwW4AyiqevgiIlJTiYqGHgUuBDoA95nZb4G7gF+4+5AE63YDYpuuLgynlTKzocCR7v58ZRsys6lmlmdmeZs2bUqwWxERqYpERUM5wCB3P2BmGcDHQG9331LTHZtZE+AeYEqiZd19BjADICcnx2u6bxER+VKiO4K97n4AwN2LgDVVSAIbgCNjxruH00q0BgYAr5nZWmAEkKsKYxGR1Ep0R9DHzJaHwwb0DscNcHcfVMm6S4AsM+tFkAC+C5xTMtPdtwMdS8bN7DXganfPq/JRiIhItSVKBH2ru2F332dmlwIvErRU+pi7rzCz6UCeu+dWd9siIlJ7EjU6V6OG5tz9BeCFMtNurGDZMTXZl4iIVE/kzutFRKRhUiIQEWnkIicCM8s0s68kMxgREUm9SInAzCYA+cBfw/FsM1Nlr4hIAxD1juBmgiYjtgG4ez5B3wQiIlLPRU0ExeFz/7H0hq+ISAMQtT+CFWZ2DpBmZlnA5cCi5IUlIiKpEvWO4DKC/or3AL8jaI76f5MUk4iIpFDUO4I+7n49cH0ygxERkdSLekdwt5mtMrNbSvolEBGRhiFSInD3sQQ9k20Cfm1m75rZDUmNTEREUiLyC2Xu/rG73wf8iOCdgnLbDBIRkfol6gtlfc3sZjN7F7if4Imh7kmNTEREUiJqZfFjwHzgRHf/KInxiIhIikVKBO4+MtmBiIhI3ag0EZjZU+5+dlgkFPsmcZQeykREpB5IdEdwRfj7tGQHIiIidaPSymJ33xgOXuzu62J/gIuTH56IiCRb1MdHx5cz7eTaDEREROpGojqCiwiu/I82s+Uxs1oDC5MZmIiIpEaiOoLfAX8BbgOujZm+w90/S1pUIiKSMokSgbv7WjO7pOwMMztMyUBEpP6LckdwGrCU4PFRi5nnwNFJiktERFKk0kTg7qeFv9UtpYhIAxW1raHRZtYyHD7XzO4xsx7JDU1ERFIh6uOjDwO7zGwwcBXwIfBk0qISEZGUiZoI9rm7A6cDD7j7gwSPkIqISD0XtfXRHWZ2HXAecLyZNQHSkxeWiIikStQ7gkkEHdf/0N0/JuiL4M6kRSUiIikTtavKj4E5QFszOw0ocvcnkhqZiIikRNSnhs4G/gl8BzgbeMvMzoqw3klm9p6ZrTaza8uZf6WZrTSz5Wb2ipkdVdUDEBGRmolaR3A98F/u/imAmXUCXgaeqWgFM0sDHiRosK4QWGJmue6+MmaxZUCOu+8K2zX6BUExlIiIpEjUOoImJUkgtCXCuscBq919jbvvBeYRPHVUyt1fdfdd4eibqB9kEZGUi3pH8FczexGYG45PAl5IsE43YH3MeCEwvJLlLyBo4O4gZjYVmArQo4feYxMRqU1R+yyeZmZnAl8NJ81w92drKwgzOxfIAb5Wwf5nADMAcnJyvLxlRESkehL1R5AF3AX0Bt4Frnb3DRG3vQE4Mma8ezit7D6+QVAH8TV33xNx2yIiUksSlfM/BjwHfJugBdL7q7DtJUCWmfUys2bAd4Hc2AXMbAjwa2BimToIERFJkURFQ63dfWY4/J6ZvR11w+6+z8wuBV4E0oDH3H2FmU0H8tw9l+CltFbA02YG8B93n1jloxARkWpLlAgywqv2kn4IMmPH3b3SxODuL1CmUtndb4wZ/kaVIxYRkVqVKBFsBO6JGf84ZtyBrycjKBERSZ1EHdOMTVUgIiJSN6K+UCYiIg2UEoGISCOnRCAi0shFbX3Uwr6KbwzHe5jZcckNTUREUiHqHcFDwEhgcji+g6BlURERqeeiNjo33N2HmtkyAHffGr4tLCIi9VzUO4LisH8Bh9L+CA4kLSoREUmZqIngPuBZoLOZ/Rz4B3Br0qISEZGUidoM9RwzWwqMI2he4gx3X5XUyEREJCUiJQIz6wHsAv4cO83d/5OswEREJDWiVhY/T1A/YEAG0At4D+ifpLhERCRFohYNDYwdN7OhwMVJiUhERFKqWm8Wh81PV9b/sIiI1BNR6wiujBltAgwFPkpKRCIiklJR6whaxwzvI6gz+H3thyMiIqmWMBGEL5K1dverUxCPiIikWKV1BGbW1N33A6NTFI+IiKRYojuCfxLUB+SbWS7wNPBFyUx3/0MSYxMRkRSIWkeQAWwh6KO45H0CB5QIRETquUSJoHP4xFABXyaAEp60qKTBKC4uprCwkKKioroORaRRyMjIoHv37qSnp0deJ1EiSANaEZ8ASigRSEKFhYW0bt2anj17Ylben5GI1BZ3Z8uWLRQWFtKrV6/I6yVKBBvdfXrNQpPGrKioSElAJEXMjA4dOrBp06YqrZfozWL990qNKQmIpE51/t8SJYJx1QtFRETqi0oTgbt/lqpARJIlLS2N7OxsBgwYwIQJE9i2bVutbHfWrFlceumltbKtnj17MnDgQLKzs8nOzmbRokW1st2y8vPzeeGFF+Km/eUvfyEnJ4d+/foxZMgQrrrqKgBuvvlm7rrrrlrb96hRo0qHp02bRv/+/Zk2bRqPPPIITzzxRI22vWzZMi644IK4aWeccQYjRoyImzZlyhSeeeaZuGmtWrUqHX7//fc55ZRTyMrKYujQoZx99tl88sknNYrts88+Y/z48WRlZTF+/Hi2bt1a7nLXXHMN/fv3p2/fvlx++eW4B9Wwc+fOZeDAgQwaNIiTTjqJzZs3A3D11Vfz97//vUaxlahWo3Mi9UlmZib5+fkUFBRw2GGH8eCDD9Z1SOV69dVXyc/PJz8/P+6kWZl9+/ZVaR9lE0FBQQGXXnopv/3tb1m5ciV5eXkcc8wxVdpmVLHJbcaMGSxfvpw777yTH/3oR3z/+9+PvJ3yjvnWW2/l8ssvLx3ftm0bS5cuZfv27axZsybSdouKijj11FO56KKL+OCDD3j77be5+OKLq1zeXtbtt9/OuHHj+OCDDxg3bhy33377QcssWrSIhQsXsnz5cgoKCliyZAkLFixg3759XHHFFbz66qssX76cQYMG8cADDwBw2WWXlbut6oj6HoFIjf30zytY+dHntbrNfke04aYJ0bvFGDlyJMuXLwfgn//8J1dccQVFRUVkZmby+OOP85WvfIVZs2aRm5vLrl27+PDDD/nWt77FL37xCwAef/xxbrvtNtq1a8fgwYNp3rw5AGvXruWHP/whmzdvplOnTjz++OP06NGDKVOmkJmZybJly/j000957LHHeOKJJ1i8eDHDhw9n1qxZFcZa2TYzMjJYtmwZo0eP5pJLLuGSSy5h06ZNtGjRgpkzZ9KnTx+efvppfvrTn5KWlkbbtm15+eWXufHGG9m9ezf/+Mc/uO6663j++ee5/vrr6dOnDxDcPV100UUHxTJz5kxmzJjB3r17OeaYY3jyySdp0aLFQft4/fXXWbFiBT/4wQ/Yu3cvBw4c4Pe//z1ZWVm0atWKnTt3MnHiRHbu3MmwYcO47rrrWLVqFa1ateLqq6/mww8/LPdYyh7zPffcUxrbjh07WL58OYMHDy6d9oc//IEJEybQpUsX5s2bx09+8pOEfxu/+93vGDlyJBMmTCidNmbMmITrJfKnP/2J1157DYDzzz+fMWPGcMcdd8QtY2YUFRWxd+9e3J3i4mK6dOmCu+PufPHFF3To0IHPP/+8NFEfddRRbNmyhY8//pjDDz+8RjHqjkAajf379/PKK68wceJEAPr06cMbb7zBsmXLmD59etzJIj8/n/nz5/Puu+8yf/581q9fz8aNG7nppptYuHAh//jHP1i5cmXp8pdddhnnn38+y5cv53vf+17c1enWrVtZvHgx9957LxMnTuTHP/4xK1as4N133yU/P790ubFjx5Kdnc3w4cMTbrOwsJBFixZxzz33MHXqVO6//36WLl3KXXfdxcUXB12FTJ8+nRdffJF33nmH3NxcmjVrxvTp05k0aRL5+flMmjSJgoIChg0blvCzO/PMM1myZAnvvPMOffv25dFHHy13HwCPPPIIV1xxBfn5+eTl5dG9e/e4beXm5pbepU2aNCluXkXHUvaYY+Xl5TFgwIC4aXPnzmXy5MlMnjyZuXPnJjw+IPJnsWPHjtIivLI/sX8TJT755BO6du0KwOGHH15uUdPIkSMZO3YsXbt2pWvXrpx44on07duX9PR0Hn74YQYOHMgRRxzBypUr44rAhg4dysKFCyMdX2V0RyApU5Ur99q0e/dusrOz2bBhA3379mX8+PEAbN++nfPPP58PPvgAM6O4uLh0nXHjxtG2bVsA+vXrx7p169i8eTNjxoyhU6dOAEyaNIn3338fgMWLF/OHPwQv2p933nlcc801pduaMGECZsbAgQPp0qULAwcG/Tz179+ftWvXkp2dDQRFQx07dixdr7Jtfuc73yEtLY2dO3eyaNEivvOd75TO27NnDwCjR49mypQpnH322Zx55pk1+gwLCgq44YYb2LZtGzt37uTEE0+scB8jR47k5z//OYWFhZx55plkZWVF2kdlxxJ7zGVt3Lix9DuB4MT7wQcf8NWvfhUzIz09nYKCAgYMGFDuEzVVfcqmdevWcQm8Ksys3P2tXr2aVatWUVhYCMD48eN54403GDFiBA8//DDLli3j6KOP5rLLLuO2227jhhtuAKBz58589FHNewRI6h2BmZ1kZu+Z2Wozu7ac+c3NbH44/y0z65nMeKRxKrn6XLduHe5eWkfwf//3f4wdO5aCggL+/Oc/x739XFLkA0FxSVXL4mOVbKtJkyZx223SpEm1t9uyZUsADhw4QLt27UrrFvLz81m1ahUQXJn/7Gc/Y/369QwbNowtW7YctJ3+/fuzdOnShPubMmUKDzzwAO+++y433XRT6WdV3j7OOeec0qv+U045JXKFZmXHEnvMZWVmZsZ9d0899RRbt26lV69e9OzZk7Vr15beFXTo0CGusvazzz4rTb5RP4uq3hF06dKFjRs3AkHS6ty580HLPPvss4wYMYJWrVrRqlUrTj75ZBYvXlyacHr37o2ZcfbZZ8fVtZQUa9ZU0hJB2Hz1g8DJQD9gspn1K7PYBcBWdz8GuBe4A5EkadGiBffddx933303+/btY/v27XTr1g2g0rL6EsOHD2fBggVs2bKF4uJinn766dJ5o0aNYt68eQDMmTOH448/vsbxRtlmmzZt6NWrV2ks7s4777wDwIcffsjw4cOZPn06nTp1Yv369bRu3ZodO3aUrj9t2jRuvfXW0jubAwcO8Mgjjxy0nx07dtC1a1eKi4uZM2dO6fTy9rFmzRqOPvpoLr/8ck4//fTSOplEKjuWyvTt25fVq1eXjs+dO5e//vWvrF27lrVr17J06dLSz3HMmDHMnz+fvXv3AsH3PnbsWADOOeccFi1axPPPP1+6rddff52CgoK4/ZXcEZT3069f2VMcTJw4kdmzZwMwe/ZsTj/99IOW6dGjR2nlcHFxMQsWLKBv375069aNlStXllZYv/TSS/Tt27d0vffff/+gYrHqSOYdwXHAandf4+57gXlA2U/gdGB2OPwMMM709pEk0ZAhQxg0aBBz587lmmuu4brrrmPIkCGRrsy7du3KzTffzMiRIxk9enTcP+T999/P448/zqBBg3jyySf51a9+VeNYo25zzpw5PProowwePJj+/fvzpz/9CQhO8gMHDmTAgAGMGjWKwYMHM3bsWFauXEl2djbz589n0KBB/PKXv2Ty5Mn07duXAQMGlPuUzS233MLw4cMZPXp0acVyRft46qmnGDBgANnZ2RQUFFTpiaCKjqUyffr0Yfv27ezYsYO1a9eybt26uMdGe/XqRdu2bXnrrbc47bTTOP744xk2bBjZ2dksXLiwtOI2MzOT5557jvvvv5+srCz69evHQw89FFfsVB3XXnstL730EllZWbz88stce21QOJKXl8eFF14IwFlnnUXv3r0ZOHAggwcPZvDgwUyYMIEjjjiCm266iRNOOIFBgwaRn59fWpdVXFzM6tWrycnJqVF8AFbyrGptM7OzgJPc/cJw/DxguLtfGrNMQbhMYTj+YbjM5jLbmgpMBejRo8ewdevWVTmev634mD/mb+Ces7PJSD+4nFGSY9WqVXEnTJFkuPfee2ndunXpibUxePbZZ3n77be55ZZbDppX3v+dmS1193KzRr14asjdZ7h7jrvnVDc7f7P/4Tz0vWFKAiIN0EUXXRRX/9IY7Nu3r/Tlv5pK5lNDG4AjY8a7h9PKW6bQzJoCbQn6PRARiSwjI4PzzjuvrsNIqdinq2oqmXcES4AsM+tlZs2A7wK5ZZbJBc4Ph88C/u7JKquSOqOvVCR1qvP/lrRE4O77gEuBF4FVwFPuvsLMppvZxHCxR4EOZrYauBI46BFTqd8yMjLYsmWLkoFICpT0R5CRkVGl9ZJWWZwsOTk5npeXV9dhSETqoUwktSrqoayyymK9WSxJlZ6eXqWekkQk9erFU0MiIpI8SgQiIo2cEoGISCNX7yqLzWwTUPVXiwMdgc0Jl2pYdMyNg465cajJMR/l7uW+kVvvEkFNmFleRbXmDZWOuXHQMTcOyTpmFQ2JiDRySgQiIo1cY0sEM+o6gDqgY24cdMyNQ1KOuVHVEYiIyMEa2x2BiIiUoUQgItLINchEYGYnmdl7ZrbazA5q0dTMmpvZ/HD+W2bWsw7CrFURjvlKM1tpZsvN7BUzO6ou4qxNiY45Zrlvm5mbWb1/1DDKMZvZ2eF3vcLMfpfqGGtbhL/tHmb2qpktC/++T6mLOGuLmT1mZp+GPTiWN9/M7L7w81huZkNrvFN3b1A/QBrwIXA00Ax4B+hXZpmLgUfC4e8C8+s67hQc81igRTh8UWM45nC51sDrwJtATl3HnYLvOQtYBrQPxzvXddwpOOYZwEXhcD9gbV3HXcNjPgEYChRUMP8U4C+AASOAt2q6z4Z4R3AcsNrd17j7XmAecHqZZU4HZofDzwDjzMxSGGNtS3jM7v6qu+8KR98k6DGuPovyPQPcAtwBNIR2sKMc838DD7r7VgB3/zTFMda2KMfsQJtwuC3wUQrjq3Xu/jrwWSWLnA484YE3gXZm1rUm+2yIiaAbsD5mvDCcVu4yHnSgsx3okJLokiPKMce6gOCKoj5LeMzhLfOR7v58KgNLoijf87HAsWa20MzeNLOTUhZdckQ55puBc82sEHgBuCw1odWZqv6/J6T+CBoZMzsXyAG+VtexJJOZNQHuAabUcSip1pSgeGgMwV3f62Y20N231WVQSTYZmOXud5vZSOBJMxvg7gfqOrD6oiHeEWwAjowZ7x5OK3cZM2tKcDu5JSXRJUeUY8bMvgFcD0x09z0pii1ZEh1za2AA8JqZrSUoS82t5xXGUb7nQiDX3Yvd/d/A+wSJob6KcswXAE8BuPtiIIOgcbaGKtL/e1U0xESwBMgys15m1oygMji3zDK5wPnh8FnA3z2shamnEh6zmQ0Bfk2QBOp7uTEkOGZ33+7uHd29p7v3JKgXmeju9bmf0yh/238kuBvAzDoSFBWtSWGMtS3KMf8HGAdgZn0JEsGmlEaZWrnA98Onh0YA2919Y0022OCKhtx9n5ldCrxI8MTBY+6+wsymA3nungs8SnD7uJqgUua7dRdxzUU85juBVsDTYb34f9x9Yp0FXUMRj7lBiXjMLwLfNLOVwH5gmrvX27vdiMd8FTDTzH5MUHE8pT5f2JnZXIJk3jGs97gJSAdw90cI6kFOAVYDu4Af1Hif9fjzEhGRWtAQi4ZERKQKlAhERBo5JQIRkUZOiUBEpJFTIhARaeSUCBoBM9tvZvkxPz0rWXZnLexvlpn9O9zX2+HbnlXdxm/MrF84/JMy8xbVNMZwOyWfS4GZ/dnM2iVYPrs6LVuaWVczey4cHmNm28P9rjKzm6qxvYklrXCa2Rkln1M4Pj18cbBGwu/wrATLvFaVF/TCY38uwnLltr5pZneZ2dej7k+iUyJoHHa7e3bMz9oU7HOau2cD1xK8yFYl7n6hu68MR39SZt6omocHfPm5DCB4n+SSBMtnEzy/XVVXAjNjxt8IP5scgjZyqtSMsLvnuvvt4egZBC1ulsy70d1frkaMh5JZQHltJN1P8PcktUyJoBEys1YW9Enwtpm9a2YHtdoZXsW+HnPFfHw4/Ztmtjhc92kza5Vgd68Dx4TrXhluq8DM/jec1tLMnjezd8Lpk8Lpr5lZjpndDmSGccwJ5+0Mf88zs1NjYp5lZmeZWZqZ3WlmSyxor/1/Inwsiwkb7jKz48JjXGZmi8zsK+FbrdOBSWEsk8LYHzOzf4bLltf6KcC3gb+WnejuXwBLgWPCu403w3ifNbP2YSyX25f9SMwLp00xswfMbBQwEbgzjKl3zGdwkpk9HfPZlF6NV/U7NLMbw8+ywMxmmMW11HtezN/IceHyUT+XclXU+qa7rwM6mNnhVdmeRFAX7W3rJ7U/BG+Y5oc/zxK8Ud4mnNeR4A3FkpcLd4a/rwKuD4fTCNru6UhwYm8ZTv9/wI3l7G8WcFY4/B3gLWAY8C7QkuAN5xXAEIKT5MyYdduGv18j7D+gJKaYZUpi/BYwOxxuRtAiYyYwFbghnN4cyAN6lRPnzpjjexo4KRxvAzQNh78B/D4cngI8ELP+rcC54XA7gnZ9WpbZRy9gacz4GOC5cLgDsBboDywHvhZOnw78Mhz+CGheso+yccR+1rHj4Xf8n5jv6mHg3Gp+h4fFTH8SmBDzHc0Mh08gbD+/os+lzLHnAL+p5G+2J+W0x09wZ/Xtuv6famg/Da6JCSnXbg+KIgAws3TgVjM7AThAcCXcBfg4Zp0lwGPhsn9093wz+xpBMcTC8KKwGcGVdHnuNLMbCNp8uYCgLZhnPbgKxsz+ABxPcKV8t5ndQXCSeKMKx/UX4Fdm1pygKOF1d99tZt8EBsWUcbclaHjt32XWzzSz/PD4VwEvxSw/28yyCJosSK9g/98EJprZ1eF4BtAj3FaJrhzc7s3xZraM4LO/naChuHbuviCcP5sgMUGQIOaY2R8J2hGKxIOmGf4KTDCzZ4BTgWsIWp2N+h2WGGtm1wAtgMMIkvifw3lzw/29bmZtLKhnqehziY0vD7gw6vHE+BQ4ohrrSSWUCBqn7wGdgGHuXmxB65wZsQuE/9gnEJxAZpnZPcBW4CV3nxxhH9Pc/ZmSETMbV95C7v5+WEZ+CvAzM3vF3adHOQh3LzKz14ATgUkEnZZA0HPTZe7+YoJN7Hb3bDNrQdCWzSXAfQSd2bzq7t+yoGL9tQrWN4Kr0/cq2wdlPluCOoLTSjdi1raS9U8luNqeAFxvZgMrWbasecClBMUsee6+IyzWifodYmYZwEMEd2frzexm4o+nbBs1TgWfi5l1qULsFckg+EylFqmOoHFqC3waJoGxwEH9F1vQp/En7j4T+A1B13lvAqPNrKTMv6WZHRtxn28AZ5hZCzNrSVCs84aZHQHscvffEjSMV17FaXF4Z1Ke+QSNbpXcXUBwUr+oZB0zOzbcZ7k86LntcuAq+7JZ8pJmfafELLqDoIisxIvAZSVl5ha08FrW+wTFHBVy9+3AVgvrYYDzgAUW9KlwpLu/SlCE05agWC1W2ZhiLSD4PP+bL5NkVb/DkpP+5rAuoeyTRCV1Ol8laAVzO9E+l+o6Fii3L1+pPiWCxmkOkGNm7wLfB/5VzjJjgHfCIoxJwK/cfRPBiXGumS0nKFLoE2WH7v42QbnzPwnqDH7j7suAgcA/wyKam4CflbP6DGC5hZXFZfyNoLjjZQ+6MoQgca0E3rbgEcRfk+DuN4xlOUEnJ78AbguPPXa9V4F+JZXFBHcO6WFsK8Lxstv9Aviw5MRbifMJitOWEzydNJ2g7uK34fe0DLjPD+5gZh4wLayU7V1m3/uB54CTw99U9TsM9zeT4OT7IkGRYayi8HN6hKAIECJ8LhY8CPCb8vZpQeubi4GvmFmhmV0QTk8nePCgPjclfkhS66MiSWZm3yIohruhrmOpz8LPcai7/19dx9LQqI5AJMnc/Vkzq899Yh8qmgJ313UQDZHuCEREGjnVEYiINHJKBCIijZwSgYhII6dEICLSyCkRiIg0cv8f4IeVdpcBm5IAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Testing results\n",
        "y_test_rfc_pred = rfc.predict(X_test_pred)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_test, y_test_rfc_pred)))\n",
        "print('Precision Score:', (precision_score(y_test, y_test_rfc_pred)))\n",
        "print('Recall Score:', (recall_score(y_test,y_test_rfc_pred)))\n",
        "print('F1 Score:', (f1_score(y_test, y_test_rfc_pred)))\n",
        "\n",
        "y_score_rf_test = rfc.predict_proba(X_test_pred)\n",
        "print('ROC Score:', (roc_auc_score(y_test, y_score_rf_test[:,1])))\n",
        "print(plot_roc_curve(rfc, X_test_pred, y_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 126,
      "metadata": {
        "id": "0O0CncQr7jMA"
      },
      "outputs": [],
      "source": [
        "#inclusion of tfidf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 127,
      "metadata": {
        "id": "kcJiZtfS7jOF"
      },
      "outputs": [],
      "source": [
        "tfidf_vect_title = TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')\n",
        "tfidf_vect_profile = TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')\n",
        "tfidf_vect_description = TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')\n",
        "tfidf_vect_requirements = TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')\n",
        "tfidf_vect_benefits = TfidfVectorizer(lowercase=False, ngram_range=(1, 4), stop_words='english')\n",
        "\n",
        "tfidfnb_title_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "tfidfnb_profile_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "tfidfnb_description_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "tfidfnb_requirements_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) \n",
        "tfidfnb_benefits_tuned = GridSearchCV(MultinomialNB(), nb_param_grid,cv = 3,scoring = 'roc_auc',verbose = 3) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 128,
      "metadata": {
        "id": "jbHMl4Tu7jQe"
      },
      "outputs": [],
      "source": [
        "X_train_title_tfidf = tfidf_vect_title.fit_transform(X_train_title_pro)\n",
        "X_test_title_tfidf = tfidf_vect_title.transform(X_test_title_pro)\n",
        "\n",
        "X_train_profile_tfidf = tfidf_vect_profile.fit_transform(X_train_profile_pro)\n",
        "X_test_profile_tfidf = tfidf_vect_profile.transform(X_test_profile_pro)\n",
        "\n",
        "X_train_description_tfidf = tfidf_vect_description.fit_transform(X_train_description_pro)\n",
        "X_test_description_tfidf = tfidf_vect_description.transform(X_test_description_pro)\n",
        "\n",
        "X_train_requirements_tfidf = tfidf_vect_requirements.fit_transform(X_train_requirements_pro)\n",
        "X_test_requirements_tfidf = tfidf_vect_requirements.transform(X_test_requirements_pro)\n",
        "\n",
        "X_train_benefits_tfidf = tfidf_vect_benefits.fit_transform(X_train_benefits_pro)\n",
        "X_test_benefits_tfidf = tfidf_vect_benefits.transform(X_test_benefits_pro)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 129,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j3Ah1jyn7jS8",
        "outputId": "cd6d38aa-3cb6-434f-84e8-2eaf71012627"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.822 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.793 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.834 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.822 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.793 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.834 total time=   0.0s\n",
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.893 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.882 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.894 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.893 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.882 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.894 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.852 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.830 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.862 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.852 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.830 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.862 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.869 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.849 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.876 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.869 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.849 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.876 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.883 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.865 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.888 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.883 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.865 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.888 total time=   0.0s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.895 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.899 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.893 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.895 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.899 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.893 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.920 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.921 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.916 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.920 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.921 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.916 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.838 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.846 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.834 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.838 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.846 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.834 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.864 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.867 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.861 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.865 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.868 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.862 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.908 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.913 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.906 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.908 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.913 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.907 total time=   0.0s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.889 total time=   0.1s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.892 total time=   0.1s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.887 total time=   0.1s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.889 total time=   0.1s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.892 total time=   0.1s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.887 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.953 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.952 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.956 total time=   0.1s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.953 total time=   0.1s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.953 total time=   0.1s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.956 total time=   0.1s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.931 total time=   0.1s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.938 total time=   0.1s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.946 total time=   0.2s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.931 total time=   0.1s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.938 total time=   0.1s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.946 total time=   0.1s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.889 total time=   0.1s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.900 total time=   0.1s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.909 total time=   0.1s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.891 total time=   0.1s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.902 total time=   0.1s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.911 total time=   0.1s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.851 total time=   0.1s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.862 total time=   0.1s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.864 total time=   0.1s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.862 total time=   0.1s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.872 total time=   0.1s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.875 total time=   0.1s\n",
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.806 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.784 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.796 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.806 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.784 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.796 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.878 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.869 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.899 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.878 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.870 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.899 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.829 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.821 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.867 total time=   0.0s\n",
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.829 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.821 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.867 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.752 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.766 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.809 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.752 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.766 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.809 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.743 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.760 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.798 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.748 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.763 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.803 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV 1/3] END ..........alpha=1, fit_prior=False;, score=0.780 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=1, fit_prior=False;, score=0.816 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=1, fit_prior=False;, score=0.814 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=1, fit_prior=True;, score=0.780 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=1, fit_prior=True;, score=0.816 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=1, fit_prior=True;, score=0.814 total time=   0.0s\n",
            "[CV 1/3] END ..........alpha=0, fit_prior=False;, score=0.804 total time=   0.0s\n",
            "[CV 2/3] END ..........alpha=0, fit_prior=False;, score=0.854 total time=   0.0s\n",
            "[CV 3/3] END ..........alpha=0, fit_prior=False;, score=0.858 total time=   0.0s\n",
            "[CV 1/3] END ...........alpha=0, fit_prior=True;, score=0.804 total time=   0.0s\n",
            "[CV 2/3] END ...........alpha=0, fit_prior=True;, score=0.854 total time=   0.0s\n",
            "[CV 3/3] END ...........alpha=0, fit_prior=True;, score=0.858 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.1, fit_prior=False;, score=0.773 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.1, fit_prior=False;, score=0.818 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.1, fit_prior=False;, score=0.815 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CV 1/3] END .........alpha=0.1, fit_prior=True;, score=0.773 total time=   0.0s\n",
            "[CV 2/3] END .........alpha=0.1, fit_prior=True;, score=0.818 total time=   0.0s\n",
            "[CV 3/3] END .........alpha=0.1, fit_prior=True;, score=0.815 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.01, fit_prior=False;, score=0.767 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.01, fit_prior=False;, score=0.817 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.01, fit_prior=False;, score=0.811 total time=   0.0s\n",
            "[CV 1/3] END ........alpha=0.01, fit_prior=True;, score=0.767 total time=   0.0s\n",
            "[CV 2/3] END ........alpha=0.01, fit_prior=True;, score=0.817 total time=   0.0s\n",
            "[CV 3/3] END ........alpha=0.01, fit_prior=True;, score=0.812 total time=   0.0s\n",
            "[CV 1/3] END ......alpha=0.001, fit_prior=False;, score=0.784 total time=   0.0s\n",
            "[CV 2/3] END ......alpha=0.001, fit_prior=False;, score=0.829 total time=   0.0s\n",
            "[CV 3/3] END ......alpha=0.001, fit_prior=False;, score=0.827 total time=   0.0s\n",
            "[CV 1/3] END .......alpha=0.001, fit_prior=True;, score=0.785 total time=   0.0s\n",
            "[CV 2/3] END .......alpha=0.001, fit_prior=True;, score=0.829 total time=   0.0s\n",
            "[CV 3/3] END .......alpha=0.001, fit_prior=True;, score=0.827 total time=   0.0s\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# fitting the model for grid search \n",
        "nb_title_tuned_tfidf = tfidfnb_title_tuned.fit(X_train_title_tfidf, y_train)\n",
        "nb_profile_tuned_tfidf = tfidfnb_profile_tuned.fit(X_train_profile_tfidf, y_train)\n",
        "nb_description_tuned_tfidf = tfidfnb_description_tuned.fit(X_train_description_tfidf, y_train)\n",
        "nb_requirements_tuned_tfidf = tfidfnb_requirements_tuned.fit(X_train_requirements_tfidf, y_train)\n",
        "nb_benefits_tuned_tfidf = tfidfnb_benefits_tuned.fit(X_train_benefits_tfidf, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 130,
      "metadata": {
        "id": "wArTlUSA8sFm"
      },
      "outputs": [],
      "source": [
        "nb_title_train_pred_tfidf = nb_title_tuned_tfidf.predict(X_train_title_tfidf)\n",
        "nb_profile_train_pred_tfidf = nb_profile_tuned_tfidf.predict(X_train_profile_tfidf)\n",
        "nb_description_train_pred_tfidf = nb_description_tuned_tfidf.predict(X_train_description_tfidf)\n",
        "nb_requirements_train_pred_tfidf = nb_requirements_tuned_tfidf.predict(X_train_requirements_tfidf)\n",
        "nb_benefits_train_pred_tfidf = nb_benefits_tuned_tfidf.predict(X_train_benefits_tfidf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 131,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PUFyr8pC8sLO",
        "outputId": "a24bbe35-8831-4105-8c81-4b76567715a7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[1, 0, 1, 1, 1],\n",
              "       [0, 0, 0, 0, 0],\n",
              "       [0, 0, 0, 0, 0],\n",
              "       ...,\n",
              "       [1, 0, 0, 0, 0],\n",
              "       [0, 0, 0, 0, 0],\n",
              "       [0, 0, 0, 0, 0]], dtype=int64)"
            ]
          },
          "execution_count": 131,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "X_train_pred_tfidf = [nb_title_train_pred_tfidf, nb_profile_train_pred_tfidf, nb_description_train_pred_tfidf, nb_description_train_pred_tfidf, nb_benefits_train_pred_tfidf]\n",
        "X_train_pred_tfidf = np.array(X_train_pred_tfidf)\n",
        "X_train_pred_tfidf = X_train_pred_tfidf.T\n",
        "#X_train_pred_tfidf = X_train_pred_tfidf[:,[0,2,3]]\n",
        "X_train_pred_tfidf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n5NQOz5y2qnh"
      },
      "source": [
        "#Ensemble Learning Second level\n",
        "\n",
        "The following models were attempted for the second level of the ensemble:\n",
        "*   Random Forest\n",
        "*   Support Vector Machine\n",
        "*   Logistic Regression\n",
        "\n",
        "These models are relatively simple and explainable which leaves most of the complexity at the first level of the ensemble."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JD0goCbB3Pzj"
      },
      "source": [
        "## Random Forest\n",
        "I found that the f1 and ROC-AUC scores were not as good as the other models."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 132,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "zR7f06099fcq",
        "outputId": "c3106178-5445-4913-8e09-e7c00007e318"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9996271439224459\n",
            "Precision Score: 0.9949811794228356\n",
            "Recall Score: 0.9974842767295597\n",
            "F1 Score: 0.9962311557788944\n",
            "ROC Score: 0.9993220605182995\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A0A6CB8EE0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqAElEQVR4nO3de5xVdb3/8ddbRAFBSESPiQQGyU0ZZZKQNMhKU8EyEzFNUo8nL+nJW5oeNSzLNCpRj0EiaghIaeH9/MprgheQEbl4QcQYtERCDhwlUD+/P9aaac+wZ/aay97TMO/n4zGPWZfvWuvz3Xtmf/f3+13r+1VEYGZmbdd2LR2AmZm1LBcEZmZtnAsCM7M2zgWBmVkb54LAzKyN276lA2ioXXfdNXr37t3SYZiZtSoLFix4JyJ65NvX6gqC3r17M3/+/JYOw8ysVZH0Rl373DRkZtbGuSAwM2vjXBCYmbVxLgjMzNo4FwRmZm1c0QoCSVMlvS1pcR37Jel6ScslLZJ0QLFiMTOzuhWzRjANOLye/V8G+qU/pwP/XcRYzMysDkV7jiAinpDUu54kRwO3RzIO9tOSuknaIyLeKlZMS9/8X9a9t7lWnDXTBFsPy51vpO7am/IN573VlrznKXy9LNfPF0P+NIXPlC2/+dJkyEvemAoPhZ7lennTtPR7t/VhzfY3ly9RputlOK6xf3P5NPq9a8R56kq3dZp/vfcui0MH7M6Qvbo16tj6tOQDZXsCq3LWK9NtWxUEkk4nqTXQq1evRl3sqeXv8I1fP9OoY83MmpvU8GN227nDNlcQZBYRk4HJAOXl5Y0qSlevex+AiccNoefHOtXYV/sNyff+5H/TVDBN7U3KkyjL9ZQnVZY/pPwxZYi7scdliqFwXvKfp/BrlyXuutIVjKmI712GP6+858r+HihDmvqvVWdMJXzvsn54Nvb/pzH/r/n/fxrxKd+CWrIgWA3slbPeM91WVMP27s6e3ToW+zJmZq1GS94+Ogf4Znr30GeA9cXsHzAzs/yKViOQNAMYCewqqRK4AmgPEBE3Aw8ARwDLgfeAbxUrFjMzq1sx7xoaV2B/AGcV6/pbXS/vPQBmZtbmnixuXV04ZmbF1+YKAjMzq8kFgZlZG9dmCoJGPshnZrbNazMFQZVW9pyHmVnRtbmCwMzMasp0+6ik7YAhwMeB94HFEfF2MQMzM7PSqLcgkPRJ4HvAF4BXgTVAB+BTkt4DfgXcFhEfFTtQMzMrjkI1gh+SzBPwH1Fr3FZJuwEnACcBtxUnvObjvmIzs/zqLQjqezo4bRr6RXMHVGx5R1Q0M2vDGt1ZLOmLzRmImZm1jKbcNXRLs0VhZmYtplBn8Zy6dgHdmz+c4vEDZWZm+RXqLD4YOBHYWGu7gAOLElGR+YEyM7OaChUETwPvRcTjtXdIerk4IZmZWSkVumvoy/XsO6T5wzEzs1JrM0NMeGIaM7P82kxBUMVdBGZmNbW5gsDMzGpyQWBm1sZlLggkXVnfupmZtU4NqREsKLD+L80PlJmZ5Ze5IIiIe+tbbzXcW2xmVkOhISYmUc8IzhFxTrNHZGZmJVXoyeL5JYnCzMxaTKEni2tMOCOpU0S8V9yQisNdBGZm+WXqI5A0XNJS4KV0fYikm4oaWZF4Yhozs5qydhb/AjgMWAsQES8AHmvIzGwb0JC7hlbV2vRhM8diZmYtoFBncZVVkg4CQlJ74FxgWfHCKgI/SGBmllfWGsG3gbOAPYE3gbJ0vdXxxDRmZjVlKggi4p2I+EZE7B4RPSLixIhYW+g4SYdLelnSckkX59nfS9KjkhZKWiTpiMZkwszMGi/rXUN7S7pX0hpJb0v6g6S9CxzTDrgR+DIwEBgnaWCtZJcBd0XE/sDxQKu8E8nMrDXL2jR0J3AXsAfwcWA2MKPAMQcCyyNiRURsBmYCR9dKE8DO6XJXkmYnMzMroawFQaeIuCMiPkh/fgN0KHDMnkDunUaV6bZcVwInSqoEHgC+k+9Ekk6XNF/S/DVr1mQMuSZ3FZuZ5VdvQSBpF0m7AA9KulhSb0mfkHQRyQd3U40DpkVET+AI4A5JW8UUEZMjojwiynv06NGkC7qv2MyspkK3jy4g+TJd9fn5Hzn7AriknmNXA3vlrPdMt+U6FTgcICLmSeoA7Aq8XSAuMzNrJoXGGurThHM/B/ST1IekADgeOKFWmr8AhwLTJA0gaW5qXNuPmZk1StYHypA0mOTun+q+gYi4va70EfGBpLOBh4F2wNSIWCJpAjA/IuYA5wNTJH2XpIYxPqI4T375eTIzs/wyFQSSrgBGkhQED5DcEvpnoM6CACAiHqBWX0JEXJ6zvBQY0aCIm0h+oszMrIasdw0dS9KE89eI+BYwhOR2TzMza+WyFgTvR8RHwAeSdibpzN2rwDFmZtYKZO0jmC+pGzCF5E6ijcC8YgVVDEXqejAza/UyFQQRcWa6eLOkh4CdI2JR8cIqHvcQmJnVVGjy+gPq2xcRzzd/SGZmVkqFagQ/q2dfAJ9vxljMzKwFFHqgbFSpAjEzs5aRearK1s5dxWZm+bWZgqCKnyczM6upzRUEZmZWU9YZyiTpREmXp+u9JB1Y3NDMzKwUstYIbgKGk8wfALCBZBrKVsPPk5mZ5Zf1yeJhEXGApIUAEbFO0g5FjKto5EfKzMxqyFoj2JJORh8AknoAHxUtKjMzK5msBcH1wD3AbpJ+RDIE9dVFi8rMzEom61hD0yUtIBmKWsBXImJZUSNrZu4iMDPLL+vENNcDMyOiVXUQ5+UuAjOzGrI2DS0ALpP0mqTrJJUXMygzMyudTAVBRNwWEUcAnwZeBq6R9GpRIzMzs5Jo6JPFfYH+wCeAl5o/HDMzK7WsTxb/NK0BTAAWA+URMbqokTUzz1BmZpZf1gfKXgOGR8Q7xQymFDzonJlZTYVmKOsfES8BzwG9JPXK3e8ZyszMWr9CNYLzgNPJP1OZZygzM9sGFJqh7PR08csRsSl3n6QORYvKzMxKJutdQ3MzbjMzs1amUB/BvwF7Ah0l7c8/n8vdGehU5NjMzKwECvURHAaMB3oCE3O2bwC+X6SYzMyshAr1EdwG3CbpaxHxuxLFZGZmJVSoaejEiPgN0FvSebX3R8TEPIeZmVkrUqizeKf0d2egS56fekk6XNLLkpZLuriONMdJWippiaQ7GxC7mZk1g0JNQ79Kf/+goSdOZzS7EfgiUAk8J2lORCzNSdMPuAQYkU5/uVtDr2NmZk3TkLGGdpbUXtKfJK2RdGKBww4ElkfEiojYDMwEjq6V5t+BGyNiHUBEvN3QDJiZWdNkfY7gSxHxv8BRwEqSUUgvLHDMnsCqnPXKdFuuTwGfkvSUpKclHZ7vRJJOlzRf0vw1a9ZkDDk/DzVkZlZT1oKgqgnpSGB2RKxvputvD/QDRgLjgCmSutVOFBGTI6I8Isp79OjRTJc2MzPIXhDcJ+klYCjwJ0k9gE0FjlkN7JWz3jPdlqsSmBMRWyLideAVkoLBzMxKJOsMZRcDB5HMQ7AF+D+2bu+v7Tmgn6Q+knYAjgfm1Erze5LaAJJ2JWkqWpE1eDMza7qsk9e3B04EDlEyoP/jwM31HRMRH0g6G3gYaAdMjYglkiYA8yNiTrrvS5KWAh8CF0bE2kbnxszMGizrxDT/DbQHbkrXT0q3nVbfQRHxAPBArW2X5ywHyVDXWz2sZmZmpZG1IPh0RAzJWX9E0gvFCMjMzEora2fxh5I+WbUiaW+SphwzM2vlstYILgQelbSC5Fb8TwDfKlpUZmZWMgULgvRW0fUkTwpXDQHxckT8o5iBmZlZadTbNCTpNGAJMAmoAHpHxCIXAmZm245CNYL/BAZFxJq0X2A6Wz8LYGZmrVihzuLNEbEGICJWADsWP6TiiGjpCMzM/jUVqhH0lHR9XesRcU5xwiqe9IE4MzNLFSoIao8wuqBYgZiZWcvIMmexmZltwwrdNTRF0uA69u0k6RRJ3yhOaGZmVgqFmoZuBC6XtC+wGFgDdCAZKnpnYCrJnURmZtZKFWoaqgCOk9QZKAf2AN4HlkXEy8UPz8zMii3TEBMRsRF4rLihmJlZS8g66JyZmW2jXBCYmbVxDSoIJHUqViBmZtYyMhUEkg5Kp5N8KV0fIummAoeZmVkrkLVG8HPgMGAtQES8ABxSrKDMzKx0MjcNRcSqWpta1QxlgUedMzPLJ+sMZaskHQSEpPbAucCy4oVVPB5yzsyspqw1gm8DZwF7AquBMuDMIsVkZmYllLVGsE9E1BhTSNII4KnmD8nMzEopa41gUsZtZmbWytRbI5A0HDgI6CHpvJxdOwPtihmYmZmVRqGmoR2Azmm6Ljnb/xc4tlhBmZlZ6RQaffRx4HFJ0yLijRLFZGZmJZS1s/g9SdcCg0jmIwAgIj5flKjMzKxksnYWTycZXqIP8ANgJfBckWIyM7MSyloQdI+IW4AtEfF4RJwCuDZgZrYNyNo0tCX9/ZakI4E3gV2KE5KZmZVS1hrBDyV1Bc4HLgB+DfxnoYMkHS7pZUnLJV1cT7qvSQpJ5RnjabDwUENmZnllnaryvnRxPTAKqp8srpOkdsCNwBeBSuA5SXMiYmmtdF1Ixi56pmGhN4482JCZWQ311ggktZM0TtIFkgan246SNBe4ocC5DwSWR8SKiNgMzASOzpPuKuAaYFPDwzczs6Yq1DR0C3Aa0B24XtJvgOuAn0bE/gWO3RPIHbq6Mt1WTdIBwF4RcX99J5J0uqT5kuavWbOmwGXNzKwhCjUNlQP7RcRHkjoAfwU+GRFrm3phSdsBE4HxhdJGxGRgMkB5eblb+83MmlGhGsHmiPgIICI2ASsaUAisBvbKWe+ZbqvSBRgMPCZpJfAZYE4xO4zNzGxrhWoE/SUtSpcFfDJdFxARsV89xz4H9JPUh6QAOB44oWpnRKwHdq1al/QYcEFEzG9wLszMrNEKFQQDGnviiPhA0tnAwyQjlU6NiCWSJgDzI2JOY89tZmbNp9Cgc00aaC4iHgAeqLXt8jrSjmzKtczMrHEyT15vZmbbJhcEZmZtXOaCQFJHSfsUMxgzMyu9TAWBpNFABfBQul4myZ29ZmbbgKw1gitJhox4FyAiKkjmJjAzs1Yua0GwJb3vP5ef8DUz2wZknY9giaQTgHaS+gHnAHOLF5aZmZVK1hrBd0jmK/4HcCfJcNT/WaSYzMyshLLWCPpHxKXApcUMxszMSi9rjeBnkpZJuqpqXgIzM9s2ZCoIImIUycxka4BfSXpR0mVFjczMzEoi8wNlEfHXiLge+DbJMwV5xwwyM7PWJesDZQMkXSnpRWASyR1DPYsamZmZlUTWzuKpwCzgsIh4s4jxmJlZiWUqCCJieLEDMTOzllFvQSDprog4Lm0Syn2SOMsMZWZm1goUqhGcm/4+qtiBmJlZy6i3szgi3koXz4yIN3J/gDOLH56ZmRVb1ttHv5hn25ebMxAzM2sZhfoIziD55r+3pEU5u7oATxUzMDMzK41CfQR3Ag8CPwYuztm+ISL+XrSozMysZAoVBBERKyWdVXuHpF1cGJiZtX5ZagRHAQtIbh9Vzr4A9i5SXGZmViL1FgQRcVT629NSmplto7KONTRC0k7p8omSJkrqVdzQzMysFLLePvrfwHuShgDnA68BdxQtKjMzK5msBcEHERHA0cANEXEjyS2kZmbWymUdfXSDpEuAk4CDJW0HtC9eWGZmVipZawRjSSauPyUi/koyF8G1RYvKzMxKJutUlX8FpgNdJR0FbIqI24samZmZlUTWu4aOA54Fvg4cBzwj6dgMxx0u6WVJyyVdnGf/eZKWSlok6U+SPtHQDJiZWdNk7SO4FPh0RLwNIKkH8Efgt3UdIKkdcCPJgHWVwHOS5kTE0pxkC4HyiHgvHdfopyTNUGZmViJZ+wi2qyoEUmszHHsgsDwiVkTEZmAmyV1H1SLi0Yh4L119Gs+DbGZWcllrBA9JehiYka6PBR4ocMyewKqc9UpgWD3pTyUZ4G4rkk4HTgfo1cvPsZmZNaescxZfKOkY4LPppskRcU9zBSHpRKAc+Fwd158MTAYoLy+PfGnMzKxxCs1H0A+4Dvgk8CJwQUSsznju1cBeOes90221r/EFkj6Iz0XEPzKe28zMmkmhdv6pwH3A10hGIJ3UgHM/B/ST1EfSDsDxwJzcBJL2B34FjKnVB2FmZiVSqGmoS0RMSZdflvR81hNHxAeSzgYeBtoBUyNiiaQJwPyImEPyUFpnYLYkgL9ExJgG58LMzBqtUEHQIf3WXjUPQcfc9Yiot2CIiAeo1akcEZfnLH+hwRGbmVmzKlQQvAVMzFn/a856AJ8vRlBmZlY6hSamGVWqQMzMrGVkfaDMzMy2US4IzMzaOBcEZmZtXNbRR5XOVXx5ut5L0oHFDc3MzEoha43gJmA4MC5d30AysqiZmbVyWQedGxYRB0haCBAR69Knhc3MrJXLWiPYks4vEFA9H8FHRYvKzMxKJmtBcD1wD7CbpB8BfwauLlpUZmZWMlmHoZ4uaQFwKMnwEl+JiGVFjczMzEoiU0EgqRfwHnBv7raI+EuxAjMzs9LI2ll8P0n/gIAOQB/gZWBQkeIyM7MSydo0tG/uuqQDgDOLEpGZmZVUo54sToefrm/+YTMzayWy9hGcl7O6HXAA8GZRIjIzs5LK2kfQJWf5A5I+g981fzhmZlZqBQuC9EGyLhFxQQniMTOzEqu3j0DS9hHxITCiRPGYmVmJFaoRPEvSH1AhaQ4wG/i/qp0RcXcRYzMzsxLI2kfQAVhLMkdx1fMEAbggMDNr5QoVBLuldwwt5p8FQJUoWlS2zdiyZQuVlZVs2rSppUMxaxM6dOhAz549ad++feZjChUE7YDO1CwAqrggsIIqKyvp0qULvXv3Rsr3Z2RmzSUiWLt2LZWVlfTp0yfzcYUKgrciYkLTQrO2bNOmTS4EzEpEEt27d2fNmjUNOq7Qk8X+77UmcyFgVjqN+X8rVBAc2rhQzMystai3IIiIv5cqELNiadeuHWVlZQwePJjRo0fz7rvvNst5p02bxtlnn90s5+rduzf77rsvZWVllJWVMXfu3GY5b20VFRU88MADNbY9+OCDlJeXM3DgQPbff3/OP/98AK688kquu+66Zrv2QQcdVL184YUXMmjQIC688EJuvvlmbr/99iade+HChZx66qk1tn3lK1/hM5/5TI1t48eP57e//W2NbZ07d65efuWVVzjiiCPo168fBxxwAMcddxx/+9vfmhTb7NmzGTRoENtttx3z58+vM91DDz3EPvvsQ9++ffnJT35Svf31119n2LBh9O3bl7Fjx7J582YAbrjhBqZOndqk2Ko0atA5s9akY8eOVFRUsHjxYnbZZRduvPHGlg4pr0cffZSKigoqKipqfGjW54MPPmjQNWoXBIsXL+bss8/mN7/5DUuXLmX+/Pn07du3QefMKrdwmzx5MosWLeLaa6/l29/+Nt/85jcznydfnq+++mrOOeec6vV3332XBQsWsH79elasWJHpvJs2beLII4/kjDPO4NVXX+X555/nzDPPbHB7e22DBw/m7rvv5pBDDqkzzYcffshZZ53Fgw8+yNKlS5kxYwZLly4F4Hvf+x7f/e53Wb58OR/72Me45ZZbADjllFOYNGlSk2KrkvU5ArMm+8G9S1j65v826zkHfnxnrhidfVqM4cOHs2jRIgCeffZZzj33XDZt2kTHjh259dZb2WeffZg2bRpz5szhvffe47XXXuOrX/0qP/3pTwG49dZb+fGPf0y3bt0YMmQIO+64IwArV67klFNO4Z133qFHjx7ceuut9OrVi/Hjx9OxY0cWLlzI22+/zdSpU7n99tuZN28ew4YNY9q0aXXGWt85O3TowMKFCxkxYgRnnXUWZ511FmvWrKFTp05MmTKF/v37M3v2bH7wgx/Qrl07unbtyh//+Ecuv/xy3n//ff785z9zySWXcP/993PppZfSv39/IKk9nXHGGVvFMmXKFCZPnszmzZvp27cvd9xxB506ddrqGk888QRLlizhW9/6Fps3b+ajjz7id7/7Hf369aNz585s3LiRMWPGsHHjRoYOHcoll1zCsmXL6Ny5MxdccAGvvfZa3rzUzvPEiROrY9uwYQOLFi1iyJAh1dvuvvtuRo8eze67787MmTP5/ve/X/Bv484772T48OGMHj26etvIkSMLHlfIgAEDCqZ59tln6du3L3vvvTcAxx9/PH/4wx8YMGAAjzzyCHfeeScAJ598MldeeSVnnHEGnTp1onfv3jz77LMceOCBTYrRNQJrMz788EP+9Kc/MWbMGAD69+/Pk08+ycKFC5kwYUKND4uKigpmzZrFiy++yKxZs1i1ahVvvfUWV1xxBU899RR//vOfq7+xAXznO9/h5JNPZtGiRXzjG9+o8e103bp1zJs3j5///OeMGTOG7373uyxZsoQXX3yRioqK6nSjRo2irKyMYcOGFTxnZWUlc+fOZeLEiZx++ulMmjSJBQsWcN1113HmmclUIRMmTODhhx/mhRdeYM6cOeywww5MmDCBsWPHUlFRwdixY1m8eDFDhw4t+Nodc8wxPPfcc7zwwgsMGDCg+ltp7WsA3HzzzZx77rlUVFQwf/58evbsWeNcc+bMqa6ljR07tsa+uvJSO8+55s+fz+DBg2tsmzFjBuPGjWPcuHHMmDGjYP6AzK/Fhg0bqpvwav/k/k00xOrVq9lrr72q13v27Mnq1atZu3Yt3bp1Y/vtt6+xvUp5eTlPPvlko66ZyzUCK5mGfHNvTu+//z5lZWWsXr2aAQMG8MUvfhGA9evXc/LJJ/Pqq68iiS1btlQfc+ihh9K1a1cABg4cyBtvvME777zDyJEj6dGjBwBjx47llVdeAWDevHncfXfyoP1JJ53ERRddVH2u0aNHI4l9992X3XffnX33TeZ5GjRoECtXrqSsrAxImoZ23XXX6uPqO+fXv/512rVrx8aNG5k7dy5f//rXq/f94x//AGDEiBGMHz+e4447jmOOOaZJr+HixYu57LLLePfdd9m4cSOHHXZYndcYPnw4P/rRj6isrOSYY46hX79+ma5RX15y81zbW2+9Vf2eAPztb3/j1Vdf5bOf/SySaN++PYsXL2bw4MF576hp6F02Xbp0qVGAt6TddtuNl156qcnnKWqNQNLhkl6WtFzSxXn27yhpVrr/GUm9ixmPtU1V3z7feOMNIqK6j+C//uu/GDVqFIsXL+bee++t8fRzVZMPJM0lDW2Lz1V1ru22267GebfbbrtGn3ennXYC4KOPPqJbt27VfQsVFRUsW7YMSL6Z//CHP2TVqlUMHTqUtWvXbnWeQYMGsWDBgoLXGz9+PDfccAMvvvgiV1xxRfVrle8aJ5xwQvW3/iOOOIJHHnkkU57qy0tunmvr2LFjjffurrvuYt26dfTp04fevXuzcuXK6lpB9+7dWbduXXXav//979WFb9bXohg1gj333JNVq1ZVr1dWVrLnnnvSvXt33n333eq/k6rtVaqaNZuqaAVBOnz1jcCXgYHAOEkDayU7FVgXEX2BnwPXFCses06dOnH99dfzs5/9jA8++ID169dX/1PV11ZfZdiwYTz++OOsXbuWLVu2MHv27Op9Bx10EDNnzgRg+vTpHHzwwU2ON8s5d955Z/r06VMdS0TwwgsvAPDaa68xbNgwJkyYQI8ePVi1ahVdunRhw4YN1cdfeOGFXH311dU1m48++oibb755q+ts2LCBPfbYgy1btjB9+vTq7fmusWLFCvbee2/OOeccjj766Oo+mULqy0t9BgwYwPLly6vXZ8yYwUMPPcTKlStZuXIlCxYsqH4dR44cyaxZs6rvvJk2bRqjRo0C4IQTTmDu3Lncf//91ed64oknWLx4cY3rVdUI8v0MHFj7Iy6bT3/607z66qu8/vrrbN68mZkzZzJmzBgkMWrUqOo7nW677TaOPvro6uNeeeWVrZrFGqOYNYIDgeURsSIiNgMzgaNrpTkauC1d/i1wqPz0kRXR/vvvz3777ceMGTO46KKLuOSSS9h///0zfTPfY489uPLKKxk+fDgjRoyo0Qk4adIkbr31Vvbbbz/uuOMOfvnLXzY51qznnD59OrfccgtDhgxh0KBB/OEPfwCSD/l9992XwYMHc9BBBzFkyBBGjRrF0qVLKSsrY9asWey333784he/YNy4cQwYMIDBgwfnvcvmqquuYtiwYYwYMaK6Y7mua9x1110MHjyYsrIyFi9e3KA7gurKS3369+/P+vXr2bBhAytXruSNN96ocdtonz596Nq1K8888wxHHXUUBx98MEOHDqWsrIynnnqKa65Jvn927NiR++67j0mTJtGvXz8GDhzITTfdVKPZqTHuueceevbsybx58zjyyCOrm9XefPNNjjjiCAC23357brjhBg477DAGDBjAcccdx6BBSVPqNddcw8SJE+nbty9r166tcZvsU089Vd3U2RSKKM6QQZKOBQ6PiNPS9ZOAYRFxdk6axWmaynT9tTTNO7XOdTpwOkCvXr2GvvHGGw2O53+W/JXfV6xm4nFldGi/dTujFceyZcsy3TVh1hQ///nP6dKlC6eddlpLh1IyCxcuZOLEidxxxx1b7cv3fydpQUSU5ztXq7hrKCImR0R5RJQ3tnT+0qB/46ZvDHUhYLYNOuOMM2r0v7QF77zzDldddVWznKuYdw2tBvbKWe+ZbsuXplLS9kBXknkPzMwy69ChAyeddFJLh1FSzdEkVKWYNYLngH6S+kjaATgemFMrzRzg5HT5WOCRKFZblbUYv6VmpdOY/7eiFQQR8QFwNvAwsAy4KyKWSJogaUya7Bagu6TlwHnAVreYWuvWoUMH1q5d68LArASq5iPo0KFDg44rWmdxsZSXl0d9AzfZvxbPUGZWWnXNUFZfZ7GfLLaiat++fYNmSjKz0msVdw2ZmVnxuCAwM2vjXBCYmbVxra6zWNIaoOGPFid2Bd4pmGrb4jy3Dc5z29CUPH8iIvI+kdvqCoKmkDS/rl7zbZXz3DY4z21DsfLspiEzszbOBYGZWRvX1gqCyS0dQAtwntsG57ltKEqe21QfgZmZba2t1QjMzKwWFwRmZm3cNlkQSDpc0suSlkvaakRTSTtKmpXuf0ZS7xYIs1llyPN5kpZKWiTpT5I+0RJxNqdCec5J9zVJIanV32qYJc+Sjkvf6yWS7ix1jM0tw992L0mPSlqY/n0f0RJxNhdJUyW9nc7gmG+/JF2fvh6LJB3Q5ItGxDb1A7QDXgP2BnYAXgAG1kpzJnBzunw8MKul4y5BnkcBndLlM9pCntN0XYAngKeB8paOuwTvcz9gIfCxdH23lo67BHmeDJyRLg8EVrZ03E3M8yHAAcDiOvYfATwICPgM8ExTr7kt1ggOBJZHxIqI2AzMBI6uleZo4LZ0+bfAoZJUwhibW8E8R8SjEfFeuvo0yYxxrVmW9xngKuAaYFsYBztLnv8duDEi1gFExNsljrG5ZclzADuny12BN0sYX7OLiCeAv9eT5Gjg9kg8DXSTtEdTrrktFgR7Aqty1ivTbXnTRDKBznqge0miK44sec51Ksk3itasYJ7TKvNeEXF/KQMroizv86eAT0l6StLTkg4vWXTFkSXPVwInSqoEHgC+U5rQWkxD/98L8nwEbYykE4Fy4HMtHUsxSdoOmAiMb+FQSm17kuahkSS1vick7RsR77ZkUEU2DpgWET+TNBy4Q9LgiPiopQNrLbbFGsFqYK+c9Z7ptrxpJG1PUp1cW5LoiiNLnpH0BeBSYExE/KNEsRVLoTx3AQYDj0laSdKWOqeVdxhneZ8rgTkRsSUiXgdeISkYWqsseT4VuAsgIuYBHUgGZ9tWZfp/b4htsSB4DugnqY+kHUg6g+fUSjMHODldPhZ4JNJemFaqYJ4l7Q/8iqQQaO3txlAgzxGxPiJ2jYjeEdGbpF9kTES05nlOs/xt/56kNoCkXUmailaUMMbmliXPfwEOBZA0gKQgWFPSKEtrDvDN9O6hzwDrI+Ktppxwm2saiogPJJ0NPExyx8HUiFgiaQIwPyLmALeQVB+Xk3TKHN9yETddxjxfC3QGZqf94n+JiDEtFnQTZczzNiVjnh8GviRpKfAhcGFEtNrabsY8nw9MkfRdko7j8a35i52kGSSF+a5pv8cVQHuAiLiZpB/kCGA58B7wrSZfsxW/XmZm1gy2xaYhMzNrABcEZmZtnAsCM7M2zgWBmVkb54LAzKyNc0HQBkj6UFJFzk/vetJubIbrTZP0enqt59OnPRt6jl9LGpguf7/WvrlNjTE9T9XrsljSvZK6FUhf1piRLSXtIem+dHmkpPXpdZdJuqIR5xtTNQqnpK9UvU7p+oT0wcEmSd/DYwukeawhD+ileb8vQ7q8o29Kuk7S57Nez7JzQdA2vB8RZTk/K0twzQsjogy4mORBtgaJiNMiYmm6+v1a+w5qenjAP1+XwSTPk5xVIH0Zyf3bDXUeMCVn/cn0tSknGSOnQcMIR8SciPhJuvoVkhE3q/ZdHhF/bESM/0qmAfnGSJpE8vdkzcwFQRskqbOSOQmel/SipK1G7Uy/xT6R84354HT7lyTNS4+dLalzgcs9AfRNjz0vPddiSf+ZbttJ0v2SXki3j023PyapXNJPgI5pHNPTfRvT3zMlHZkT8zRJx0pqJ+laSc8pGa/9PzK8LPNIB+6SdGCax4WS5kraJ32qdQIwNo1lbBr7VEnPpmnzjX4K8DXgodobI+L/gAVA37S28XQa7z2SPpbGco7+OY/EzHTbeEk3SDoIGANcm8b0yZzX4HBJs3Nem+pv4w19DyVdnr6WiyVNlmqM1HtSzt/IgWn6rK9LXnWNvhkRbwDdJf1bQ85nGbTEeNv+Ke0PyROmFenPPSRPlO+c7tuV5AnFqocLN6a/zwcuTZfbkYzdsyvJB/tO6fbvAZfnud404Nh0+evAM8BQ4EVgJ5InnJcA+5N8SE7JObZr+vsx0vkDqmLKSVMV41eB29LlHUhGZOwInA5clm7fEZgP9MkT58ac/M0GDk/Xdwa2T5e/APwuXR4P3JBz/NXAielyN5JxfXaqdY0+wIKc9ZHAfelyd2AlMAhYBHwu3T4B+EW6/CawY9U1aseR+1rnrqfv8V9y3qv/Bk5s5Hu4S872O4DROe/RlHT5ENLx8+t6XWrlvRz4dT1/s73JMx4/Sc3qay39P7Wt/WxzQ0xYXu9H0hQBgKT2wNWSDgE+IvkmvDvw15xjngOmpml/HxEVkj5H0gzxVPqlcAeSb9L5XCvpMpIxX04lGQvmnki+BSPpbuBgkm/KP5N0DcmHxJMNyNeDwC8l7UjSlPBERLwv6UvAfjlt3F1JBl57vdbxHSVVpPlfBvy/nPS3SepHMmRB+zqu/yVgjKQL0vUOQK/0XFX2YOtxbw6WtJDktf8JyUBx3SLi8XT/bSQFEyQFxHRJvycZRyiTSIZmeAgYLem3wJHARSSjzmZ9D6uMknQR0AnYhaQQvzfdNyO93hOSdlbSz1LX65Ib33zgtKz5yfE28PFGHGf1cEHQNn0D6AEMjYgtSkbn7JCbIP3HPoTkA2SapInAOuD/RcS4DNe4MCJ+W7Ui6dB8iSLilbSN/Ajgh5L+FBETsmQiIjZJegw4DBhLMmkJJDM3fSciHi5wivcjokxSJ5KxbM4CrieZzObRiPiqko71x+o4XiTfTl+u7xrUem1J+giOqj6J1LWe448k+bY9GrhU0r71pK1tJnA2STPL/IjYkDbrZH0PkdQBuImkdrZK0pXUzE/tMWqCOl4XSbs3IPa6dCB5Ta0ZuY+gbeoKvJ0WAqOAreYvVjKn8d8iYgrwa5Kp854GRkiqavPfSdKnMl7zSeArkjpJ2omkWedJSR8H3ouI35AMjJev43RLWjPJZxbJoFtVtQtIPtTPqDpG0qfSa+YVycxt5wDn65/DklcN6zs+J+kGkiayKg8D36lqM1cywmttr5A0c9QpItYD65T2wwAnAY8rmVNhr4h4lKQJpytJs1qu2jHlepzk9fx3/llINvQ9rPrQfyftS6h9J1FVn85nSUbBXE+216WxPgXkncvXGs8FQds0HSiX9CLwTeClPGlGAi+kTRhjgV9GxBqSD8YZkhaRNCn0z3LBiHiepN35WZI+g19HxEJgX+DZtInmCuCHeQ6fDCxS2llcy/+QNHf8MZKpDCEpuJYCzyu5BfFXFKj9prEsIpnk5KfAj9O85x73KDCwqrOYpObQPo1tSbpe+7z/B7xW9cFbj5NJmtMWkdydNIGk7+I36fu0ELg+tp5gZiZwYdop+8la1/4QuA/4cvqbhr6H6fWmkHz4PkzSZJhrU/o63UzSBAgZXhclNwL8Ot81lYy+OQ/YR1KlpFPT7e1JbjxozUOJ/0vy6KNmRSbpqyTNcJe1dCytWfo6HhAR/9XSsWxr3EdgVmQRcY+k1jwn9r+K7YGftXQQ2yLXCMzM2jj3EZiZtXEuCMzM2jgXBGZmbZwLAjOzNs4FgZlZG/f/AVIeLYTaB8ZdAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Training results\n",
        "rfc2 = RandomForestClassifier(oob_score = True, criterion = \"entropy\")\n",
        "result = rfc2.fit(X_train_pred_tfidf,y_train)\n",
        "y_train_rfc_pred_tfidf = rfc2.predict(X_train_pred_tfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_train, y_train_rfc_pred_tfidf)))\n",
        "print('Precision Score:', (precision_score(y_train, y_train_rfc_pred_tfidf)))\n",
        "print('Recall Score:', (recall_score(y_train,y_train_rfc_pred_tfidf)))\n",
        "print('F1 Score:', (f1_score(y_train, y_train_rfc_pred_tfidf)))\n",
        "\n",
        "y_score_rf_tfidf = rfc2.predict_proba(X_train_pred_tfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_train, y_score_rf_tfidf[:,1])))\n",
        "print(plot_roc_curve(rfc2, X_train_pred_tfidf, y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 133,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 559
        },
        "id": "eb5CGGcF8sOy",
        "outputId": "b546793b-83f4-4fae-9994-5627592e56c5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0 0 0 0 0]\n",
            " [1 0 0 0 0]\n",
            " [1 0 1 1 1]\n",
            " ...\n",
            " [0 0 0 0 0]\n",
            " [0 0 0 0 0]\n",
            " [0 0 0 0 0]]\n",
            "Accuracy Score: 0.9865771812080537\n",
            "Precision Score: 0.9795918367346939\n",
            "Recall Score: 0.676056338028169\n",
            "F1 Score: 0.8\n",
            "ROC Score: 0.8229100871976178\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A125C0B7F0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAwR0lEQVR4nO3deXxV1bn/8c/DGIYAlalURFCjjBIhFdHaQil1BKtVEauVqpdbJ2yt+NPqVYvV2qo4oNaCIk4FtNVbnG9tFRVQCRIggANSlCgtg4AgUyDP74+9z+EkOeTsQM4Jyfm+X6+8cvb87BPYa69n7b2WuTsiIpK9GtR2ACIiUrtUEIiIZDkVBCIiWU4FgYhIllNBICKS5RrVdgDV1a5dO+/atWtthyEiUqfMmzdvrbu3T7aszhUEXbt2pbCwsLbDEBGpU8zs0z0tU2pIRCTLqSAQEclyKghERLKcCgIRkSyngkBEJMulrSAws8lmttrMivew3MzsPjNbZmYLzaxfumIREZE9S2eNYApwYhXLTwLywp/RwB/TGIuIiOxB2goCd38T+LKKVU4DHvfAO0AbM+uUrnhEROqqsjLn1heX8Nm6LWnZf222ERwIrEyYLgnnVWJmo82s0MwK16xZk5HgRET2Fw+/vZxJb/2L2Z+sTcv+60RjsbtPdPcCdy9o3z7pG9IiIvVS8ecbuePVDzmhV0dGfPugtByjNguCz4HEs+oczhMREWDLjp2MmTqfti2acvsZR2JmaTlObRYEM4Cfhk8PHQNsdPdVtRiPiMh+5ZYXlvCvdV8zfkRfvtGiSdqOk7ZO58xsKjAIaGdmJcBNQGMAd38IeAk4GVgGbAF+lq5YRETqmleKVzH1vZX8/HuHcuyh7dJ6rLQVBO4+MsVyBy5L1/FFROqqVRu38v/+uogjO7fmqqGHp/14daKxWEQkW+wqc66avoDSXWXce85RNGmU/st0nRuPQESkPpv45nLmLF/HH848km7tWmTkmKoRiIjsJxas3MBd//chp/TpxFn9O2fsuCoIRET2A19v38kvphfRIbcpt53eJ22Piiaj1JCIyH7gN88vZsW6r5n2X8fQunnjjB5bNQIRkVr24sJVPF1YwmWDDmPAIW0zfvxINQIzawD0Bb4FbAWK3X11OgMTEckGn2/YynXPLiT/oDZc+YO8WomhyoLAzA4F/h/wA+BjYA2QAxxuZluAPwGPuXtZugMVEalvdpU5v5xexK4y595z8mncsHaSNKlqBL8lGCfgv8MXwOLMrANwLnA+8Fh6whMRqb/++MYy3vvXl9x1Vl8ObpuZR0WTqbIgqOrt4DA1dE9NByQikg3mf7aeu1/7mOF9v8UZ/ZL2wJ8xe10PMbOhNRmIiEi22Lx9J1dOK+KbrXL47em9M/qoaDL7kpB6pMaiEBHJIjf+rZiS9Vu495x8WuVk9lHRZFI1Fs/Y0yIg8884iYjUcTMWfMGz73/OlUPyKOh6QG2HA6RuLD4eOA/YXGG+AUenJSIRkXpq5ZdbuP65RfQ/+Btc8f3DajucuFQFwTvAFnefWXGBmX2YnpBEROqfnbvK+OX0InC4Z0Q+jWrpUdFkUj01dFIVy75b8+GIiNQv23fuYs2m7Tz5zmcUfrqee8/J56ADmtd2WOWoryERkWpydzZt38maTdtZ/dV2Vm/aFnzetD38vS2cv52NW0vj251+1IGcll+7j4omo4JARCRUVuas+3rH7ot57ML+1TbWbN4ev7iv3rSNbaWVO1Ro2qgBHVo1pX3LphzaviUDD21Lh9ymdMjNoWPrHI47dP98xkYFgYjUe7H0zO479u2s+WpbeFHffRe/dvMOdpV5pe1b5TSiQ6scOuQ25agubeIX99hFv0OrprTPzaFVTqNafydgb6ggEJE6yd3ZvH1ncDFPSM/ELvSJ6ZoNW0orbd/AoG3LpuFFvSk9OuXGL+4dcoMLe/C7KTmNG9bCGWZO5ILAzG5295v3NC0iUhPKypwvt+yIX9xjd+wVc+9rNm1na+muSts3adQgfnE/pF1LBnQL0zOtgrv49uGyti2b0rBB3bt7T4fq1AjmpZgWEdmj7Tt3sXbzDlYnpmSS5N73lJ7JzWkUT8nE0jPtYyma3LqfnqlNkQsCd3++qmkRyT6x9MzudEzYsFohRZMqPRPLs8fSM7G79sS7+PqenqlNqbqYmABULppD7j6mxiMSkVqXmJ4J7ti3lWtUjV/kv6o6PdM+tynd2rUol55JvIs/oEWT/erFqmyVqkZQmJEoRCQjduwsS3Jh386aMPceS9Os3bydnSnSM307t0mae++Qm0OrZkrP1CWp3iwuN+CMmTV39y3pDUlEqmvz9p3lc++xO/evyj9Bsz5JesYM2rbYnYo5omNu/OIev4tvGVzomzVReqY+ijpm8UCCbqdbAl3MrC/BqGWXpjM4kWxWVuas37KjXO690hM04ectO5KkZxo2CO7SW+1OzyTLvbdVeibrRW0svgc4AZgB4O4LzEx9DYnshR07y1i7ufzFffcFfls8977H9EzTRrQPn3WPpWfaJ1zclZ6R6qrOU0MrK/yjqnwLIpLFYumZck/QhOmZ3Y9IbqsyPRO7Y4+lZ4KnaXZf3JWekXSIWhCsNLNjATezxsCVwNL0hSWyf6iYnkl8qaliiqaq9Ez73KYc3LY53+72jfhde/zpmVZKz0jtiloQ/By4FzgQ+AJ4FbgsXUGJpFvF9EziS01rKrzNmio90yf29ExCw2qsm4LWzRorPSP7vUgFgbuvBX5S3Z2b2YkEBUhD4GF3v73C8i7AY0CbcJ1r3f2l6h5HJObreN8zSZ6gCXPvazZv58uvd1TaNkjPNIn3MXN4x9yEC3z5u3ilZ6Q+ifrU0CEEF/RjCF4wmwP80t2XV7FNQ+ABYChQAsw1sxnuviRhtRuAp939j2bWE3gJ6Lo3JyL1V1mZs2Frabl+Zsr1//5V9dIzBV2/Ua5zsfjTMy2b0FjpGclCUVNDfya4qJ8eTp8DTAUGVLHN0cCyWGFhZtOA04DEgsCBVuHn1gRpJ8kSpbvKKndF8FXlJ2jWbt5O6a7K6ZmWTRvF79L7JD49k5B7V3pGJLWoBUFzd38iYfpJMxubYpsDgZUJ0yVULjhuBv7PzK4AWgA/SLYjMxsNjAbo0qVLxJCltiSmZyp2KJZ44a8qPdMufFomLyE9075cF8FNad5EvaiL1IRUfQ0dEH582cyuBaYR3MWPIEjj7KuRwBR3vyt8ae0JM+vt7uWG/nH3icBEgIKCgj32fSTp4+6s31Ja7omZxA7F1iT0SfN1kvRM44ZGh9wc2uU25aADmtP/4N3pmViHYx1yc5SeEakFqW6p5hFc+GP16v9OWObAdVVs+zlwUMJ053BeoouAEwHcfY6Z5QDtgNUp4pIaUrorfHomSe49uOCHd/ER0jO9vtWKwUd0KHfXHntUsk1zpWdE9lep+hrqtg/7ngvkmVk3ggLgHODcCut8BgwBpphZDyAHWLMPx5TQlh07y13cV1d4qamq9AzEnp4J0jOHdcgt17CaeBev9IxI3VedEcp6Az0JLtYAuPvje1rf3Xea2eUE7xw0BCa7+2IzGwcUuvsM4FfAJDP7JUENY5S7K/WzB7H0TMWRmso9QbOp6vRM+5ZNad8qJ56eqTywR1PatWyq9IxIFrEo110zuwkYRFAQvAScBLzt7memNbokCgoKvLCwfvWOHUvPxJ5zL5d7rzCSU7L0TIsmDenQqnw3wBU7F4s9PdNAQ/OJZCUzm+fuBcmWRa0RnAn0Bea7+8/MrCPwZE0FWF8lpmcSuyKoOND2l1t2kKw8jqVn2uc25bD27So1rMby8C2aKj0jInsv6hVkq7uXmdlOM2tF0Jh7UKqN6iN3Z8OW0qQNq7EL/dpw3ubtOyttH0/P5Dal8zea0+/gbyS9i1d6RkQyJWpBUGhmbYBJBE8SbSZ4u7je2LmrLBhYu8JLTfG7+GqkZ3p8qxXfyy1/1x67i2+j9IyI7Gei9jUUG4DmITN7BWjl7gvTF1Z6rP5qGy8tWpXQRfDugbb3lJ45oEWT+MX80PZtyzWsJt7FKz0jInVVqhfK+lW1zN3fr/mQ0ufBNz5hyuwVNGpg8Qt4LD2TLPfermVTmjRSekZE6rdUt7F3VbHMge/XYCxp9+XXO+hyQHPeuHqQ0jMiIqFUL5QNzlQgmfDVtlLaNFeOXkQkUVblPTZuLaVVTuPaDkNEZL+SVQXBV1tLad1MBYGISKLsKgi27aRVMz3dIyKSKFJBYIHzzOzGcLqLmR2d3tBqnlJDIiKVRa0RPAgMJBg/AGATwYhldca20l3s2FlGK6WGRETKiZonGeDu/cxsPoC7rzezJmmMq8Z9ta0UQAWBiEgFUWsEpeFg9A5gZu2Bsqo32b98tTUsCHLURiAikihqQXAf8BzQwcxuBd4GbktbVGmwcWvQAZyeGhIRKS9qX0NPmdk8gtHEDPiRuy9Na2Q1LNYTaEv1CSQiUk6kq6KZ3QdMc/c61UCcKDYAj94qFhEpL2pqaB5wg5l9YmZ3mlnSUW5ERKTuiVQQuPtj7n4y8G3gQ+D3ZvZxWiMTEZGMqO6bxYcB3YGDgQ9qPhwREcm0qG8W/yGsAYwDioECdx+W1shERCQjoj5C8wkw0N3XpjMYERHJvFQjlHV39w+AuUAXM+uSuLyujVAmIiKVpaoRXAWMJvlIZXVuhDIREaks1Qhlo8OPJ7n7tsRlZpaTtqhERCRjoj41NDviPBERqWNStRF8EzgQaGZmRxF0LwHQCmie5thERCQDUrURnACMAjoD4xPmbwJ+naaYREQkg1K1ETwGPGZmP3b3v2YoJhERyaBUqaHz3P1JoKuZXVVxubuPT7KZiIjUIakai1uEv1sCuUl+qmRmJ5rZh2a2zMyu3cM6Z5vZEjNbbGZ/rkbsIiJSA1Klhv4U/v5NdXccjmj2ADAUKAHmmtkMd1+SsE4ecB1wXDj8ZYfqHkdERPZNdfoaamVmjc3sH2a2xszOS7HZ0cAyd1/u7juAacBpFdb5L+ABd18P4O6rq3sCIiKyb6K+R/BDd/8KOBVYQdAL6dgU2xwIrEyYLgnnJTocONzMZpnZO2Z2YrIdmdloMys0s8I1a9ZEDFlERKKIWhDEUkinAM+4+8YaOn4jIA8YBIwEJplZm4oruftEdy9w94L27dvX0KFFRASiFwQvmNkHQH/gH2bWHtiWYpvPgYMSpjuH8xKVADPcvdTd/wV8RFAwiIhIhkQdoexa4FiCcQhKga+pnO+vaC6QZ2bdzKwJcA4wo8I6/0tQG8DM2hGkipZHDV5ERPZd1MHrGwPnAd81M4CZwENVbePuO83scuBVoCEw2d0Xm9k4oNDdZ4TLfmhmS4BdwFh3X7fXZyMiItUWdWCaPwKNgQfD6fPDeRdXtZG7vwS8VGHejQmfnaCr60ovq4mISGZELQi+7e59E6b/aWYL0hGQiIhkVtTG4l1mdmhswswOIUjliIhIHRe1RjAWeN3MlhN0RX0w8LO0RSUiIhmTsiAIHxXdSPCmcKwLiA/dfXs6AxMRkcyoMjVkZhcDi4EJQBHQ1d0XqhAQEak/UtUIfgH0cvc1YbvAU1R+F0BEROqwVI3FO9x9DYC7Lweapj8kERHJpFQ1gs5mdt+ept19THrCEhGRTElVEFTsYXReugIREZHaEWXMYhERqcdSPTU0ycx672FZCzO70Mx+kp7QREQkE1Klhh4AbjSzPkAxsAbIIegquhUwmeBJIhERqaNSpYaKgLPNrCVQAHQCtgJL3f3D9IcnIiLpFqmLCXffDLyR3lBERKQ2RO10TkRE6ikVBCIiWa5aBYGZNU9XICIiUjsiFQRmdmw4nOQH4XRfM3swxWYiIlIHRK0R3A2cAKwDcPcFwHfTFZSIiGRO5NSQu6+sMEsjlImI1ANRRyhbaWbHAm5mjYErgaXpC0tERDIlao3g58BlwIHA50A+cGmaYhIRkQyKWiM4wt3L9SlkZscBs2o+JBERyaSoNYIJEeeJiEgdU2WNwMwGAscC7c3sqoRFrYCG6QxMREQyI1VqqAnQMlwvN2H+V8CZ6QpKREQyJ1XvozOBmWY2xd0/zVBMIiKSQVEbi7eY2R1AL4LxCABw9++nJSoREcmYqI3FTxF0L9EN+A2wApibpphERCSDohYEbd39EaDU3We6+4WAagMiIvVA1NRQafh7lZmdAnwBHJCekEREJJOi1gh+a2atgV8BVwMPA79ItZGZnWhmH5rZMjO7tor1fmxmbmYFEeMREZEaEnWoyhfCjxuBwRB/s3iPzKwh8AAwFCgB5prZDHdfUmG9XIK+i96tXugiIlITqqwRmFlDMxtpZlebWe9w3qlmNhu4P8W+jwaWuftyd98BTANOS7LeLcDvgW3VD19ERPZVqtTQI8DFQFvgPjN7ErgT+IO7H5Vi2wOBxK6rS8J5cWbWDzjI3V+sakdmNtrMCs2scM2aNSkOKyIi1ZEqNVQAHOnuZWaWA/wbONTd1+3rgc2sATAeGJVqXXefCEwEKCgo8H09toiI7JaqRrDD3csA3H0bsLwahcDnwEEJ053DeTG5QG/gDTNbARwDzFCDsYhIZqWqEXQ3s4XhZwMODacNcHc/sopt5wJ5ZtaNoAA4Bzg3ttDdNwLtYtNm9gZwtbsXVvssRERkr6UqCHrs7Y7dfaeZXQ68StBT6WR3X2xm44BCd5+xt/sWEZGak6rTuX3qaM7dXwJeqjDvxj2sO2hfjiUiInsn8uD1IiJSP6kgEBHJcpELAjNrZmZHpDMYERHJvEgFgZkNA4qAV8LpfDNTY6+ISD0QtUZwM0GXERsA3L2IYGwCERGp46IWBKXhc/+J9IaviEg9EHU8gsVmdi7Q0MzygDHA7PSFJSIimRK1RnAFwXjF24E/E3RH/Ys0xSQiIhkUtUbQ3d2vB65PZzAiIpJ5UWsEd5nZUjO7JTYugYiI1A+RCgJ3H0wwMtka4E9mtsjMbkhrZCIikhGRXyhz93+7+33AzwneKUjaZ5CIiNQtUV8o62FmN5vZImACwRNDndMamYiIZETUxuLJwHTgBHf/Io3xiIhIhkUqCNx9YLoDERGR2lFlQWBmT7v72WFKKPFN4igjlImISB2QqkZwZfj71HQHIiIitaPKxmJ3XxV+vNTdP038AS5Nf3giIpJuUR8fHZpk3kk1GYiIiNSOVG0ElxDc+R9iZgsTFuUCs9IZmIiIZEaqNoI/Ay8DvwOuTZi/yd2/TFtUIiKSMakKAnf3FWZ2WcUFZnaACgMRkbovSo3gVGAeweOjlrDMgUPSFJeIiGRIlQWBu58a/tawlCIi9VTUvoaOM7MW4efzzGy8mXVJb2giIpIJUR8f/SOwxcz6Ar8CPgGeSFtUaaABlkVEkotaEOx0dwdOA+539wcIHiGtO8KSwKpeS0Qk60TtfXSTmV0HnA8cb2YNgMbpC6vmeVgSmKkoEBFJFLVGMIJg4PoL3f3fBGMR3JG2qNLAVSMQEUkq6lCV/waeAlqb2anANnd/PK2R1bB4QaCSQESknKhPDZ0NvAecBZwNvGtmZ0bY7kQz+9DMlpnZtUmWX2VmS8xsoZn9w8wOru4JRBVrLDbVCUREyonaRnA98G13Xw1gZu2B14C/7GkDM2sIPEDQYV0JMNfMZrj7koTV5gMF7r4l7NfoDwRpqBrnHmsjSMfeRUTqrqhtBA1ihUBoXYRtjwaWuftyd98BTCN46ijO3V939y3h5DukcRxkPT4qIpJc1BrBK2b2KjA1nB4BvJRimwOBlQnTJcCAKta/iKCDu0rMbDQwGqBLl717j01tBCIiyUUds3ismZ0BfCecNdHdn6upIMzsPKAA+N4ejj8RmAhQUFCwlzf3YWpIbQQiIuWkGo8gD7gTOBRYBFzt7p9H3PfnwEEJ053DeRWP8QOCNojvufv2iPvea6oRiIiUlyrPPxl4AfgxQQ+kE6qx77lAnpl1M7MmwDnAjMQVzOwo4E/A8AptEDXO1UggIpJUqtRQrrtPCj9/aGbvR92xu+80s8uBV4GGwGR3X2xm44BCd59B8FJaS+CZ8I3fz9x9eLXPIko84W/VCEREyktVEOSEd+2xy2ezxGl3r7JgcPeXqNCo7O43Jnz+QbUj3ku73yxWSSAikihVQbAKGJ8w/e+EaQe+n46g0mF3X0O1HIiIyH4m1cA0gzMVSLqpryERkeSivlBW56mNQEQkuewpCHx3b0MiIrJb1hQEMaoRiIiUF7X3UQvHKr4xnO5iZkenN7SapTYCEZHkotYIHgQGAiPD6U0EPYvWGRqhTEQkuaidzg1w935mNh/A3deHbwvXGaoRiIgkF7VGUBqOL+AQH4+gLG1RpYF6HxURSS5qQXAf8BzQwcxuBd4GbktbVGmgEcpERJKL2g31U2Y2DxhCkF35kbsvTWtkNUwjlImIJBepIDCzLsAW4PnEee7+WboCq2nqfFREJLmojcUvElxLDcgBugEfAr3SFFfNUxuBiEhSUVNDfRKnzawfcGlaIkoTPT4qIpLcXr1ZHHY/XdX4w/sdPT4qIpJc1DaCqxImGwD9gC/SElGaqNM5EZHkorYR5CZ83knQZvDXmg8nfTQwjYhIcikLgvBFslx3vzoD8aSNBqYREUmuyjYCM2vk7ruA4zIUT9qojUBEJLlUNYL3CNoDisxsBvAM8HVsobs/m8bY0kMlgYhIOVHbCHKAdQRjFMfeJ3CgzhQEeqFMRCS5VAVBh/CJoWJ2FwAxdevaGutiQlWCjCotLaWkpIRt27bVdigiWSEnJ4fOnTvTuHHjyNukKggaAi1JnlCpUwWBHh+tHSUlJeTm5tK1a1e9zCeSZu7OunXrKCkpoVu3bpG3S1UQrHL3cfsW2v5BjcW1Y9u2bSoERDLEzGjbti1r1qyp1nap3iyuN/97d/c+Wm9Oqc7Qdy6SOXvz/y1VQTBk70LZ/+wej0BERBJVWRC4+5eZCiTdNEJZ9mrYsCH5+fn07t2bYcOGsWHDhhrZ75QpU7j88strZF9du3alT58+5Ofnk5+fz+zZs2tkvxUVFRXx0ksvlZv38ssvU1BQQM+ePTnqqKP41a9+BcDNN9/MnXfeWWPHPvbYY+Ofx44dS69evRg7diwPPfQQjz/++D7te/78+Vx00UXl5v3oRz/imGOOKTdv1KhR/OUvfyk3r2XLlvHPH330ESeffDJ5eXn069ePs88+m//85z/7FNuXX37J0KFDycvLY+jQoaxfvz7petdccw29evWiR48ejBkzBndny5YtnHLKKXTv3p1evXpx7bXXxte///77mTx58j7FFrNXnc7VRRqhLHs1a9aMoqIiiouLOeCAA3jggQdqO6SkXn/9dYqKiigqKip30azKzp07q3WMigVBcXExl19+OU8++SRLliyhsLCQww47rFr7jCqxcJs4cSILFy7kjjvu4Oc//zk//elPI+8n2TnfdtttjBkzJj69YcMG5s2bx8aNG1m+fHmk/W7bto1TTjmFSy65hI8//pj333+fSy+9tNr59opuv/12hgwZwscff8yQIUO4/fbbK60ze/ZsZs2axcKFCykuLmbu3LnMnDkTgKuvvpoPPviA+fPnM2vWLF5++WUALrzwQiZMmLBPscVEfY+gznO1Fte63zy/mCVffFWj++z5rVbcNCz6sBgDBw5k4cKFALz33ntceeWVbNu2jWbNmvHoo49yxBFHMGXKFGbMmMGWLVv45JNPOP300/nDH/4AwKOPPsrvfvc72rRpQ9++fWnatCkAK1as4MILL2Tt2rW0b9+eRx99lC5dujBq1CiaNWvG/PnzWb16NZMnT+bxxx9nzpw5DBgwgClTpuwx1qr2mZOTw/z58znuuOO47LLLuOyyy1izZg3Nmzdn0qRJdO/enWeeeYbf/OY3NGzYkNatW/Paa69x4403snXrVt5++22uu+46XnzxRa6//nq6d+8OBLWnSy65pFIskyZNYuLEiezYsYPDDjuMJ554gubNm1c6xptvvsnixYv52c9+xo4dOygrK+Ovf/0reXl5tGzZks2bNzN8+HA2b95M//79ue6661i6dCktW7bk6quv5pNPPkl6LhXPefz48fHYNm3axMKFC+nbt2983rPPPsuwYcPo2LEj06ZN49e//nXKfxt//vOfGThwIMOGDYvPGzRoUMrtUvnb3/7GG2+8AcAFF1zAoEGD+P3vf19uHTNj27Zt7NixA3entLSUjh070rx5cwYPHgxAkyZN6NevHyUlJQA0b96crl278t5773H00UfvU4xZUyOIUWooe+3atYt//OMfDB8+HIDu3bvz1ltvMX/+fMaNG1fuYlFUVMT06dNZtGgR06dPZ+XKlaxatYqbbrqJWbNm8fbbb7NkyZL4+ldccQUXXHABCxcu5Cc/+Um5u9P169czZ84c7r77boYPH84vf/lLFi9ezKJFiygqKoqvN3jwYPLz8xkwYEDKfZaUlDB79mzGjx/P6NGjmTBhAvPmzePOO+/k0kuDoULGjRvHq6++yoIFC5gxYwZNmjRh3LhxjBgxgqKiIkaMGEFxcTH9+/dP+d2dccYZzJ07lwULFtCjRw8eeeSRpMcAeOihh7jyyispKiqisLCQzp07l9vXjBkz4rW0ESNGlFu2p3OpeM6JCgsL6d27d7l5U6dOZeTIkYwcOZKpU6emPD8g8nexadOmeAqv4k/iv4mY//znP3Tq1AmAb37zm0lTTQMHDmTw4MF06tSJTp06ccIJJ9CjR49y62zYsIHnn3+eIUN2N90WFBTw1ltvRTq/qmRRjSD4rXKg9lTnzr0mbd26lfz8fD7//HN69OjB0KFDAdi4cSMXXHABH3/8MWZGaWlpfJshQ4bQunVrAHr27Mmnn37K2rVrGTRoEO3btwdgxIgRfPTRRwDMmTOHZ58NXrQ///zzueaaa+L7GjZsGGZGnz596NixI336BOM89erVixUrVpCfnw8EqaF27drFt6tqn2eddRYNGzZk8+bNzJ49m7POOiu+bPv27QAcd9xxjBo1irPPPpszzjhjn77D4uJibrjhBjZs2MDmzZs54YQT9niMgQMHcuutt1JSUsIZZ5xBXl5epGNUdS6J51zRqlWr4n8TCC68H3/8Md/5zncwMxo3bkxxcTG9e/dO+kRNdZ+yyc3NLVeAV4eZJT3esmXLWLp0afxuf+jQobz11lscf/zxQJAOGzlyJGPGjOGQQw6Jb9ehQwc++OCDvYolUVprBGZ2opl9aGbLzOzaJMubmtn0cPm7ZtY1XbFohLLsFbv7/PTTT3H3eBvB//zP/zB48GCKi4t5/vnny739HEv5QJAuqW4uPlFsXw0aNCi33wYNGuz1flu0aAFAWVkZbdq0ibctFBUVsXTpUiC4M//tb3/LypUr6d+/P+vWrau0n169ejFv3ryUxxs1ahT3338/ixYt4qabbop/V8mOce6558bv+k8++WT++c9/Rjqnqs4l8ZwratasWbm/3dNPP8369evp1q0bXbt2ZcWKFfFaQdu2bcs11n755Zfxwjfqd1HdGkHHjh1ZtWoVEBRaHTp0qLTOc889xzHHHEPLli1p2bIlJ510EnPmzIkvHz16NHl5efziF78ot10srbmv0lYQhN1XPwCcBPQERppZzwqrXQSsd/fDgLuB35MmqhFI8+bNue+++7jrrrvYuXMnGzdu5MADDwSoMlcfM2DAAGbOnMm6desoLS3lmWeeiS879thjmTZtGgBPPfVU/E5uX0TZZ6tWrejWrVs8FndnwYIFAHzyyScMGDCAcePG0b59e1auXElubi6bNm2Kbz927Fhuu+22eM2mrKyMhx56qNJxNm3aRKdOnSgtLeWpp56Kz092jOXLl3PIIYcwZswYTjvttHibTCpVnUtVevTowbJly+LTU6dO5ZVXXmHFihWsWLGCefPmxb/HQYMGMX36dHbs2AEEf/dYDv7cc89l9uzZvPjii/F9vfnmmxQXF5c7XqxGkOynZ8+KlzgYPnw4jz32GACPPfYYp512WqV1unTpwsyZM9m5cyelpaXMnDkznhq64YYb2LhxI/fcc0+l7T766KNKabG9kc4awdHAMndf7u47gGlAxW/gNOCx8PNfgCGWplt2dTEhAEcddRRHHnkkU6dO5ZprruG6667jqKOOinRn3qlTJ26++WYGDhzIcccdVy6HO2HCBB599FGOPPJInnjiCe699959jjXqPp966ikeeeQR+vbtS69evfjb3/4GBBf5Pn360Lt3b4499lj69u3L4MGDWbJkCfn5+UyfPp0jjzySe+65h5EjR9KjRw969+6d9CmbW265hQEDBnDcccfFG5b3dIynn36a3r17k5+fT3FxcbWeCNrTuVSle/fubNy4kU2bNrFixQo+/fTTco+NduvWjdatW/Puu+9y6qmncvzxx9O/f3/y8/OZNWtWvOG2WbNmvPDCC0yYMIG8vDx69uzJgw8+WC7ttDeuvfZa/v73v5OXl8drr70WfwS0sLCQiy++GIAzzzyTQw89lD59+tC3b1/69u3LsGHDKCkp4dZbb2XJkiX069eP/Px8Hn744fi+Z82aFU917guLP01Tw8zsTOBEd784nD4fGODulyesUxyuUxJOfxKus7bCvkYDowG6dOnS/9NPP612PP+3+N/8regLxo/oS9NGlfOMkh5Lly6t1OglUtPuvvtucnNz4xfWbDB//nzGjx/PE088UWlZsv93ZjbP3QuS7atOPDXk7hPdvcDdC/a2dP5hr2/ywE/6qRAQqYcuueSScu0v2WDt2rXccsstNbKvdD419DlwUMJ053BesnVKzKwR0Jpg3AMRkchycnI4//zzazuMjKqJlFBMOmsEc4E8M+tmZk2Ac4AZFdaZAVwQfj4T+KenK1cltUZ/UpHM2Zv/b2krCNx9J3A58CqwFHja3Reb2TgzGx6u9gjQ1syWAVcBlR4xlbotJyeHdevWqTAQyYDYeAQ5OTnV2i5tjcXpUlBQ4IWFhbUdhkSkEcpEMmtPI5RV1VicNW8WS+1o3LhxtUZKEpHMqxNPDYmISPqoIBARyXIqCEREslydayw2szVA9V8tDrQD1qZcq37ROWcHnXN22JdzPtjdk76RW+cKgn1hZoV7ajWvr3TO2UHnnB3Sdc5KDYmIZDkVBCIiWS7bCoKJtR1ALdA5Zwedc3ZIyzlnVRuBiIhUlm01AhERqUAFgYhIlquXBYGZnWhmH5rZMjOr1KOpmTU1s+nh8nfNrGsthFmjIpzzVWa2xMwWmtk/zOzg2oizJqU654T1fmxmbmZ1/lHDKOdsZmeHf+vFZvbnTMdY0yL82+5iZq+b2fzw3/fJtRFnTTGzyWa2OhzBMdlyM7P7wu9joZn12+eDunu9+gEaAp8AhwBNgAVAzwrrXAo8FH4+B5he23Fn4JwHA83Dz5dkwzmH6+UCbwLvAAW1HXcG/s55wHzgG+F0h9qOOwPnPBG4JPzcE1hR23Hv4zl/F+gHFO9h+cnAy4ABxwDv7usx62ON4Ghgmbsvd/cdwDTgtArrnAY8Fn7+CzDErE4Pa5/ynN39dXffEk6+QzBiXF0W5e8McAvwe6A+9IMd5Zz/C3jA3dcDuPvqDMdY06KcswOtws+tgS8yGF+Nc/c3gS+rWOU04HEPvAO0MbNO+3LM+lgQHAisTJguCeclXceDAXQ2Am0zEl16RDnnRBcR3FHUZSnPOawyH+TuL2YysDSK8nc+HDjczGaZ2TtmdmLGokuPKOd8M3CemZUALwFXZCa0WlPd/+8paTyCLGNm5wEFwPdqO5Z0MrMGwHhgVC2HkmmNCNJDgwhqfW+aWR9331CbQaXZSGCKu99lZgOBJ8yst7uX1XZgdUV9rBF8DhyUMN05nJd0HTNrRFCdXJeR6NIjyjljZj8ArgeGu/v2DMWWLqnOORfoDbxhZisIcqkz6niDcZS/cwkww91L3f1fwEcEBUNdFeWcLwKeBnD3OUAOQeds9VWk/+/VUR8LgrlAnpl1M7MmBI3BMyqsMwO4IPx8JvBPD1th6qiU52xmRwF/IigE6nreGFKcs7tvdPd27t7V3bsStIsMd/e6PM5plH/b/0tQG8DM2hGkipZnMMaaFuWcPwOGAJhZD4KCYE1Go8ysGcBPw6eHjgE2uvuqfdlhvUsNuftOM7sceJXgiYPJ7r7YzMYBhe4+A3iEoPq4jKBR5pzai3jfRTznO4CWwDNhu/hn7j681oLeRxHPuV6JeM6vAj80syXALmCsu9fZ2m7Ec/4VMMnMfknQcDyqLt/YmdlUgsK8XdjucRPQGMDdHyJoBzkZWAZsAX62z8esw9+XiIjUgPqYGhIRkWpQQSAikuVUEIiIZDkVBCIiWU4FgYhIllNBkAXMbJeZFSX8dK1i3c01cLwpZvav8Fjvh297VncfD5tZz/Dzryssm72vMYb7iX0vxWb2vJm1SbF+/t70bGlmnczshfDzIDPbGB53qZndtBf7Gx7rhdPMfhT7nsLpceGLg/sk/BuemWKdN6rzgl547i9EWC9p75tmdqeZfT/q8SQ6FQTZYau75yf8rMjAMce6ez5wLcGLbNXi7he7+5Jw8tcVlh277+EBu7+X3gTvk1yWYv18gue3q+sqYFLC9Fvhd1NA0EdOtboRdvcZ7n57OPkjgh43Y8tudPfX9iLG/ckUIFkfSRMI/j1JDVNBkIXMrKUFYxK8b2aLzKxSr53hXeybCXfMx4fzf2hmc8JtnzGzlikO9yZwWLjtVeG+is3sF+G8Fmb2opktCOePCOe/YWYFZnY70CyM46lw2ebw9zQzOyUh5ilmdqaZNTSzO8xsrgX9tf93hK9lDmHHXWZ2dHiO881stpkdEb7VOg4YEcYyIox9spm9F66brPdTgB8Dr1Sc6e5fA/OAw8LaxjthvM+Z2TfCWMbY7nEkpoXzRpnZ/WZ2LDAcuCOM6dCE7+BEM3sm4buJ341X929oZjeG32WxmU00K9dT7/kJ/0aODteP+r0ktafeN939U6CtmX2zOvuTCGqjv239ZPaH4A3TovDnOYI3yluFy9oRvKEYe7lwc/j7V8D14eeGBH33tCO4sLcI5/8/4MYkx5sCnBl+Pgt4F+gPLAJaELzhvBg4iuAiOSlh29bh7zcIxw+IxZSwTizG04HHws9NCHpkbAaMBm4I5zcFCoFuSeLcnHB+zwAnhtOtgEbh5x8Afw0/jwLuT9j+NuC88HMbgn59WlQ4RjdgXsL0IOCF8HNbYAXQC1gIfC+cPw64J/z8BdA0doyKcSR+14nT4d/4s4S/1R+B8/byb3hAwvwngGEJf6NJ4efvEvafv6fvpcK5FwAPV/FvtitJ+uMnqFn9uLb/T9W3n3rXxYQktdWDVAQAZtYYuM3MvguUEdwJdwT+nbDNXGByuO7/unuRmX2PIA0xK7wpbEJwJ53MHWZ2A0GfLxcR9AXznAd3wZjZs8DxBHfKd5nZ7wkuEm9V47xeBu41s6YEqYQ33X2rmf0QODIhx92aoOO1f1XYvpmZFYXnvxT4e8L6j5lZHkGXBY33cPwfAsPN7OpwOgfoEu4rphOV+7053szmE3z3txN0FNfG3WeGyx8jKJggKCCeMrP/JehHKBIPumZ4BRhmZn8BTgGuIeh1NurfMGawmV0DNAcOICjEnw+XTQ2P96aZtbKgnWVP30tifIXAxVHPJ8Fq4Ft7sZ1UQQVBdvoJ0B7o7+6lFvTOmZO4Qvgf+7sEF5ApZjYeWA/83d1HRjjGWHf/S2zCzIYkW8ndPwpz5CcDvzWzf7j7uCgn4e7bzOwN4ARgBMGgJRCM3HSFu7+aYhdb3T3fzJoT9GVzGXAfwWA2r7v76RY0rL+xh+2N4O70w6qOQYXvlqCN4NT4TsxaV7H9KQR328OA682sTxXrVjQNuJwgzVLo7pvCtE7UvyFmlgM8SFA7W2lmN1P+fCr2UePs4Xsxs47ViH1Pcgi+U6lBaiPITq2B1WEhMBioNH6xBWMa/8fdJwEPEwyd9w5wnJnFcv4tzOzwiMd8C/iRmTU3sxYEaZ23zOxbwBZ3f5KgY7xkDaelYc0kmekEnW7FahcQXNQviW1jZoeHx0zKg5HbxgC/st3dkse69R2VsOomghRZzKvAFbGcuQU9vFb0EUGaY4/cfSOw3sJ2GOB8YKYFYyoc5O6vE6RwWhOk1RJVjCnRTILv87/YXUhW928Yu+ivDdsSKj5JFGvT+Q5BL5gbifa97K3DgaRj+creU0GQnZ4CCsxsEfBT4IMk6wwCFoQpjBHAve6+huDCONXMFhKkFLpHOaC7v0+Qd36PoM3gYXefD/QB3gtTNDcBv02y+URgoYWNxRX8H0G64zUPhjKEoOBaArxvwSOIfyJF7TeMZSHBICd/AH4Xnnvidq8DPWONxQQ1h8ZhbIvD6Yr7/Rr4JHbhrcIFBOm0hQRPJ40jaLt4Mvw7zQfu88oDzEwDxoaNsodWOPYu4AXgpPA31f0bhsebRHDxfZUgZZhoW/g9PUSQAoQI34sFDwI8nOyYFvS+OQc4wsxKzOyicH5jggcP6nJX4vsl9T4qkmZmdjpBGu6G2o6lLgu/x37u/j+1HUt9ozYCkTRz9+fMrC6Pib2/aATcVdtB1EeqEYiIZDm1EYiIZDkVBCIiWU4FgYhIllNBICKS5VQQiIhkuf8PCLCzSKR2x1UAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "nb_title_test_pred_tfidf = nb_title_tuned_tfidf.predict(X_test_title_tfidf)\n",
        "nb_profile_test_pred_tfidf = nb_profile_tuned_tfidf.predict(X_test_profile_tfidf)\n",
        "nb_description_test_pred_tfidf = nb_description_tuned_tfidf.predict(X_test_description_tfidf)\n",
        "nb_requirements_test_pred_tfidf = nb_requirements_tuned_tfidf.predict(X_test_requirements_tfidf)\n",
        "nb_benefits_test_pred_tfidf = nb_benefits_tuned_tfidf.predict(X_test_benefits_tfidf)\n",
        "\n",
        "X_test_pred_tfidf = [nb_title_test_pred_tfidf, nb_profile_test_pred_tfidf, nb_description_test_pred_tfidf, nb_description_test_pred_tfidf, nb_benefits_test_pred_tfidf]\n",
        "X_test_pred_tfidf = np.array(X_test_pred_tfidf)\n",
        "X_test_pred_tfidf = X_test_pred_tfidf.T\n",
        "#X_test_pred_tfidf = X_test_pred_tfidf[:,[0,2,3]]\n",
        "print(X_test_pred_tfidf)\n",
        "\n",
        "#Test results\n",
        "y_test_rfc_pred_tfidf = rfc2.predict(X_test_pred_tfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_test, y_test_rfc_pred_tfidf)))\n",
        "print('Precision Score:', (precision_score(y_test, y_test_rfc_pred_tfidf)))\n",
        "print('Recall Score:', (recall_score(y_test,y_test_rfc_pred_tfidf)))\n",
        "print('F1 Score:', (f1_score(y_test, y_test_rfc_pred_tfidf)))\n",
        "\n",
        "y_score_rf_tfidf = rfc2.predict_proba(X_test_pred_tfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_test, y_score_rf_tfidf[:,1])))\n",
        "print(plot_roc_curve(rfc2, X_test_pred_tfidf, y_test))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i4aNu1HELixj"
      },
      "source": [
        "Accuracy Score: 0.9854586129753915\n",
        "Precision Score: 0.953125\n",
        "Recall Score: 0.7261904761904762\n",
        "F1 Score: 0.8243243243243242\n",
        "ROC Score: 0.908824474625531"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 134,
      "metadata": {
        "id": "xBm8OQj2-0gK"
      },
      "outputs": [],
      "source": [
        "x_train_bowandtfidf =  np.hstack((X_train_pred[:,[0, 1, 2,3, 4]], X_train_pred_tfidf[:,[0, 1, 2,3, 4]]))\n",
        "rfc3 = RandomForestClassifier(oob_score = True, criterion = 'gini')\n",
        "result3 = rfc3.fit(x_train_bowandtfidf,y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 135,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "36ye68j7ATPR",
        "outputId": "7fafdc79-f633-4103-f822-dadfebef82ed"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9996271439224459\n",
            "Precision Score: 0.9949811794228356\n",
            "Recall Score: 0.9974842767295597\n",
            "F1 Score: 0.9962311557788944\n",
            "ROC Score: 0.999326829817825\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A15511F0A0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAp/ElEQVR4nO3de5xVVf3/8ddbRAFBSES/JhIYJDdllElC0iArDQXLVMQ0Sf36zUtaKqbpVw3LMpVK1AwSUUNASwvv3195TfACMiIXL6gYg5pIyhe+SqB8fn/sPdOZ4TBnz+WcaZj38/GYx+zL2nt/1jkzZ5211t5rKSIwM7PWa5vmDsDMzJqXCwIzs1bOBYGZWSvngsDMrJVzQWBm1spt29wB1NfOO+8cPXv2bO4wzMxalPnz578bEd3y7WtxBUHPnj2ZN29ec4dhZtaiSHpjS/vcNGRm1sq5IDAza+VcEJiZtXIuCMzMWjkXBGZmrVzRCgJJUyW9I2nRFvZL0rWSlklaKGm/YsViZmZbVswawTTg0Dr2fxXok/6cCvy6iLGYmdkWFO05goh4XFLPOpIcAdwayTjYT0nqImm3iHirWDG9+Pb/snrdhlpx1kwTbD4sd76Rumtvyjec92Zb8p6n8PWyXD9fDPnTFD5TtvzmSxMZ0uSLqfBQ6A2+3mbvb7brN9l7l/d69T9P/pgKx53/NWngcZtvatB71/D/sXxpsp1r8zQN/Vut//XzhtPA4f8P7rcrg/bo0qBj69KcD5TtDqzIWa9Mt21WEEg6laTWQI8ePRp0sSeXvcs3f/t0g441M2tqUv2P2WXHdltdQZBZREwGJgOUl5c3qChd+d6HAFxz9CD22KlDjX2135B870/+N00F09TepDyJslxPeVJl+UPKH1OGuBt6XIbz5Mtxtveg8GvX0Ljz2SymIr53GV+mBr4HTRd33k0NeO+y/o812XtX6r+5hnzKN6PmLAhWAnvkrHdPtxXV5z7dld27tC/2ZczMWozmvH10NvCt9O6hzwFritk/YGZm+RWtRiBpBjAc2FlSJXAp0BYgIm4E7gdGAsuAD4BvFysWMzPbsmLeNTS2wP4AzijW9Te7Xv6+ezOzVq/VPVncsrpwzMyKr9UVBGZmVpMLAjOzVq7VFAQNfJDPzGyr12oKgiot7DkPM7Oia3UFgZmZ1ZTp9lFJ2wCDgE8CHwKLIuKdYgZmZmalUWdBIOnTwA+ALwGvAKuAdsBnJH0A/Aa4JSI2FTvQxnIXgZlZfoVqBD8mmSfgv6LWuK2SdgGOA04AbilOeE0v70BaZmatWJ0FQV1PB6dNQ79s6oDMzKy0GtxZLOnLTRmImZk1j8bcNXRTk0VRAn6OwMwsv0KdxbO3tAvo2vThFJ+fIzAzq6lQZ/GBwPHAulrbBexflIjMzKykChUETwEfRMRjtXdIeqk4IZmZWSkVumvoq3XsO6jpwykez0dgZpZfqxtiwl0EZmY1tbqCwMzManJBYGbWyrWagsDPEZiZ5Ze5IJB0WV3rLYY7CczMaqhPjWB+gXUzM2uBMhcEEXFPXetmZtYyFRpiYhJ1DOUfEWc1eURF4i4CM7P8Cj1ZPK8kUZSQ5yMwM6up0JPFNSackdQhIj4obkhmZlZKmfoIJA2VtAR4MV0fJOmGokZmZmYlkbWz+JfAIcBqgIh4HmhRYw35QQIzs/zqc9fQilqbPm7iWErC8xGYmdVUqLO4ygpJBwAhqS1wNrC0eGGZmVmpZK0RfAc4A9gdeBMoS9fNzKyFy1QQRMS7EfHNiNg1IrpFxPERsbrQcZIOlfSSpGWSLsizv4ekRyQtkLRQ0siGZCIL9xCYmeWX9a6hPSXdI2mVpHck/UnSngWOaQNcD3wV6A+MldS/VrKLgTsiYl/gWKDodyK5i8DMrKasTUO3A3cAuwGfBO4EZhQ4Zn9gWUS8FhEbgJnAEbXSBLBjutyZpNnJzMxKKGtB0CEibouIj9Kf3wHtChyzO5B7p1Flui3XZcDxkiqB+4Hv5juRpFMlzZM0b9WqVRlDNjOzLOosCCTtJGkn4AFJF0jqKelTks4n+eBurLHAtIjoDowEbpO0WUwRMTkiyiOivFu3bg26kB8jMDPLr9Dto/NJmm+qmtb/K2dfABfWcexKYI+c9e7ptlwnA4cCRMRcSe2AnYF3CsTVYPKDBGZmNRQaa6hXI879LNBHUi+SAuBY4Lhaaf4GHAxMk9SPpLnJbT9mZiWU9YEyJA0kufunum8gIm7dUvqI+EjSmcBDQBtgakQsljQBmBcRs4FzgSmSvk9SwxgX4UYcM7NSylQQSLoUGE5SENxPckvoX4EtFgQAEXE/tfoSIuKSnOUlwLB6RdxALl/MzPLLetfQUSRNOG9HxLeBQSS3e7Y47iEwM6spa0HwYURsAj6StCNJZ+4eBY4xM7MWIGsfwTxJXYApJHcSrQPmFisoMzMrnUwFQUScni7eKOlBYMeIWFi8sJqeewjMzPIrNHn9fnXti4jnmj6k4vJjBGZmNRWqEVxTx74AvtiEsZiZWTMo9EDZiFIFYmZmzSPzVJVmZrZ1ajUFgZ8nMzPLr9UUBFXkR8rMzGrIOkOZJB0v6ZJ0vYek/YsbmpmZlULWGsENwFCS+QMA1pJMQ2lmZi1c1ieLh0TEfpIWAETEe5K2K2JcTc5dBGZm+WWtEWxMJ6MPAEndgE1Fi6qY3EVgZlZD1oLgWuBuYBdJPyEZgvqKokVlZmYlk3WsoemS5pMMRS3gaxGxtKiRmZlZSWSdmOZaYGZEtNgOYk9MY2aWX9amofnAxZJelXS1pPJiBlVMHnTOzKymTAVBRNwSESOBzwIvAVdKeqWokZmZWUnU98ni3kBf4FPAi00fjpmZlVrWJ4t/ntYAJgCLgPKIGFXUyMzMrCSyPlD2KjA0It4tZjBmZlZ6hWYo6xsRLwLPAj0k9cjd3xJnKDMzs5oK1QjOAU4l/0xlnqHMzGwrUGiGslPTxa9GxPrcfZLaFS0qMzMrmax3Dc3JuM3MzFqYQn0E/wHsDrSXtC//GrJtR6BDkWMzM7MSKNRHcAgwDugOTMzZvhb4YZFiMjOzEirUR3ALcIukb0TEH0oUU1F5hAkzs5oKNQ0dHxG/A3pKOqf2/oiYmOcwMzNrQQp1Fu+Q/u4IdMrzUydJh0p6SdIySRdsIc0xkpZIWizp9nrEbmZmTaBQ09Bv0t8/qu+J0xnNrge+DFQCz0qaHRFLctL0AS4EhqXTX+5S3+uYmVnj1GesoR0ltZX0F0mrJB1f4LD9gWUR8VpEbABmAkfUSvOfwPUR8R5ARLxT3wyYmVnjZH2O4CsR8b/A4cByklFIxxc4ZndgRc56Zbot12eAz0h6UtJTkg7NdyJJp0qaJ2neqlWrMoZsZmZZZC0IqpqQDgPujIg1TXT9bYE+wHBgLDBFUpfaiSJickSUR0R5t27dmujSZmYG2QuCeyW9CAwG/iKpG7C+wDErgT1y1run23JVArMjYmNEvA68TFIwmJlZiWSdoewC4ACSeQg2Av/H5u39tT0L9JHUS9J2wLHA7Fpp/khSG0DSziRNRa9lDd7MzBov6+T1bYHjgYOUTPr7GHBjXcdExEeSzgQeAtoAUyNisaQJwLyImJ3u+4qkJcDHwPiIWN3g3JiZWb1lnZjm10Bb4IZ0/YR02yl1HRQR9wP319p2Sc5ykAx1vdnDak0tothXMDNrmbIWBJ+NiEE56w9Ler4YARVbWqMxM7NU1s7ijyV9umpF0p4kTTlmZtbCZa0RjAcekfQaybhtnwK+XbSozMysZAoWBOmtomtInhSuGgLipYj4ZzEDMzOz0qizaUjSKcBiYBJQAfSMiIUuBMzMth6FagTfAwZExKq0X2A6mz8LYGZmLVihzuINEbEKICJeA7YvfkhmZlZKhWoE3SVdu6X1iDirOGGZmVmpFCoIao8wOr9YgZiZWfPIMmexmZltxQrdNTRF0sAt7NtB0kmSvlmc0JpW4DEmzMzyKdQ0dD1wiaS9gUXAKqAdyVDROwJTSe4kajE8wISZWU2FmoYqgGMkdQTKgd2AD4GlEfFS8cMzM7NiyzTERESsAx4tbihmZtYcsg46Z2ZmWykXBGZmrVy9CgJJHYoViJmZNY9MBYGkA9LpJF9M1wdJuqHAYWZm1gJkrRH8AjgEWA0QEc8DBxUrKDMzK53MTUMRsaLWJs9QZma2Fcg6Q9kKSQcAIaktcDawtHhhmZlZqWStEXwHOAPYHVgJlAGnFykmMzMroaw1gr0iosaYQpKGAU82fUjFER5qyMwsr6w1gkkZt/3bkwcbMjOroc4agaShwAFAN0nn5OzaEWhTzMDMzKw0CjUNbQd0TNN1ytn+v8BRxQrKzMxKp9Doo48Bj0maFhFvlCgmMzMroaydxR9IugoYQDIfAQAR8cWiRGVmZiWTtbN4OsnwEr2AHwHLgWeLFJOZmZVQ1oKga0TcBGyMiMci4iTAtQEzs61A1qahjenvtyQdBrwJ7FSckMzMrJSy1gh+LKkzcC5wHvBb4HuFDpJ0qKSXJC2TdEEd6b4hKSSVZ4zHzMyaSNapKu9NF9cAI6D6yeItktQGuB74MlAJPCtpdkQsqZWuE8nYRU/XL3QzM2sKddYIJLWRNFbSeZIGptsOlzQHuK7AufcHlkXEaxGxAZgJHJEn3eXAlcD6+odvZmaNVahp6CbgFKArcK2k3wFXAz+PiH0LHLs7kDt0dWW6rZqk/YA9IuK+uk4k6VRJ8yTNW7VqVYHLmplZfRRqGioH9omITZLaAW8Dn46I1Y29sKRtgInAuEJpI2IyMBmgvLzcw8eZmTWhQjWCDRGxCSAi1gOv1aMQWAnskbPePd1WpRMwEHhU0nLgc8BsdxibmZVWoRpBX0kL02UBn07XBURE7FPHsc8CfST1IikAjgWOq9oZEWuAnavWJT0KnBcR8+qdCzMza7BCBUG/hp44Ij6SdCbwEMlIpVMjYrGkCcC8iJjd0HObmVnTKTToXKMGmouI+4H7a227ZAtphzfmWmZm1jCZJ683M7OtkwsCM7NWLnNBIKm9pL2KGYyZmZVepoJA0iigAngwXS+T5M5eM7OtQNYawWUkQ0a8DxARFSRzE5iZWQuXtSDYmN73n8tP+JqZbQWyzkewWNJxQBtJfYCzgDnFC8vMzEola43guyTzFf8TuJ1kOOrvFSkmMzMroaw1gr4RcRFwUTGDMTOz0staI7hG0lJJl1fNS2BmZluHTAVBRIwgmZlsFfAbSS9IuriokZmZWUlkfqAsIt6OiGuB75A8U5B3zCAzM2tZsj5Q1k/SZZJeACaR3DHUvaiRmZlZSWTtLJ4KzAIOiYg3ixiPmZmVWKaCICKGFjsQMzNrHnUWBJLuiIhj0iah3CeJs8xQZmZmLUChGsHZ6e/Dix2ImZk1jzo7iyPirXTx9Ih4I/cHOL344ZmZWbFlvX30y3m2fbUpAzEzs+ZRqI/gNJJv/ntKWpizqxPwZDEDMzOz0ijUR3A78ADwU+CCnO1rI+IfRYvKzMxKplBBEBGxXNIZtXdI2smFgZlZy5elRnA4MJ/k9lHl7AtgzyLFZWZmJVJnQRARh6e/PS2lmdlWKutYQ8Mk7ZAuHy9poqQexQ3NzMxKIevto78GPpA0CDgXeBW4rWhRmZlZyWQtCD6KiACOAK6LiOtJbiE1M7MWLuvoo2slXQicABwoaRugbfHCMjOzUslaIxhDMnH9SRHxNslcBFcVLSozMyuZrFNVvg1MBzpLOhxYHxG3FjUyMzMriax3DR0DPAMcDRwDPC3pqAzHHSrpJUnLJF2QZ/85kpZIWijpL5I+Vd8MmJlZ42TtI7gI+GxEvAMgqRvwZ+D3WzpAUhvgepIB6yqBZyXNjoglOckWAOUR8UE6rtHPSZqhzMysRLL2EWxTVQikVmc4dn9gWUS8FhEbgJkkdx1Vi4hHIuKDdPUpPA+ymVnJZa0RPCjpIWBGuj4GuL/AMbsDK3LWK4EhdaQ/mWSAu81IOhU4FaBHDz/HZmbWlLLOWTxe0pHA59NNkyPi7qYKQtLxQDnwhS1cfzIwGaC8vDzypTEzs4YpNB9BH+Bq4NPAC8B5EbEy47lXAnvkrHdPt9W+xpdI+iC+EBH/zHhuMzNrIoXa+acC9wLfIBmBdFI9zv0s0EdSL0nbAccCs3MTSNoX+A0wulYfhJmZlUihpqFOETElXX5J0nNZTxwRH0k6E3gIaANMjYjFkiYA8yJiNslDaR2BOyUB/C0iRtc7F2Zm1mCFCoJ26bf2qnkI2ueuR0SdBUNE3E+tTuWIuCRn+Uv1jtjMzJpUoYLgLWBizvrbOesBfLEYQZmZWekUmphmRKkCMTOz5pH1gTIzM9tKuSAwM2vlXBCYmbVyWUcfVTpX8SXpeg9J+xc3NDMzK4WsNYIbgKHA2HR9LcnIomZm1sJlHXRuSETsJ2kBQES8lz4tbGZmLVzWGsHGdH6BgOr5CDYVLSozMyuZrAXBtcDdwC6SfgL8FbiiaFGZmVnJZB2Gerqk+cDBJMNLfC0ilhY1MjMzK4lMBYGkHsAHwD252yLib8UKzMzMSiNrZ/F9JP0DAtoBvYCXgAFFisvMzEoka9PQ3rnrkvYDTi9KRGZmVlINerI4HX66rvmHzcyshcjaR3BOzuo2wH7Am0WJyMzMSiprH0GnnOWPSPoM/tD04ZiZWakVLAjSB8k6RcR5JYjHzMxKrM4+AknbRsTHwLASxWNmZiVWqEbwDEl/QIWk2cCdwP9V7YyIu4oYm5mZlUDWPoJ2wGqSOYqrnicIwAWBmVkLV6gg2CW9Y2gR/yoAqkTRorKtxsaNG6msrGT9+vXNHYpZq9CuXTu6d+9O27ZtMx9TqCBoA3SkZgFQxQWBFVRZWUmnTp3o2bMnUr4/IzNrKhHB6tWrqayspFevXpmPK1QQvBURExoXmrVm69evdyFgViKS6Nq1K6tWrarXcYWeLPZ/rzWaCwGz0mnI/1uhguDghoViZmYtRZ0FQUT8o1SBmBVLmzZtKCsrY+DAgYwaNYr333+/Sc47bdo0zjzzzCY5V8+ePdl7770pKyujrKyMOXPmNMl5a6uoqOD++++vse2BBx6gvLyc/v37s++++3LuuecCcNlll3H11Vc32bUPOOCA6uXx48czYMAAxo8fz4033sitt97aqHMvWLCAk08+uca2r33ta3zuc5+rsW3cuHH8/ve/r7GtY8eO1csvv/wyI0eOpE+fPuy3334cc8wx/P3vf29UbHfeeScDBgxgm222Yd68eVtM9+CDD7LXXnvRu3dvfvazn1Vvf/311xkyZAi9e/dmzJgxbNiwAYDrrruOqVOnNiq2Kg0adM6sJWnfvj0VFRUsWrSInXbaieuvv765Q8rrkUceoaKigoqKihofmnX56KOP6nWN2gXBokWLOPPMM/nd737HkiVLmDdvHr17967XObPKLdwmT57MwoULueqqq/jOd77Dt771rcznyZfnK664grPOOqt6/f3332f+/PmsWbOG1157LdN5169fz2GHHcZpp53GK6+8wnPPPcfpp59e7/b22gYOHMhdd93FQQcdtMU0H3/8MWeccQYPPPAAS5YsYcaMGSxZsgSAH/zgB3z/+99n2bJlfOITn+Cmm24C4KSTTmLSpEmNiq1K1ucIzBrtR/csZsmb/9uk5+z/yR25dFT2aTGGDh3KwoULAXjmmWc4++yzWb9+Pe3bt+fmm29mr732Ytq0acyePZsPPviAV199la9//ev8/Oc/B+Dmm2/mpz/9KV26dGHQoEFsv/32ACxfvpyTTjqJd999l27dunHzzTfTo0cPxo0bR/v27VmwYAHvvPMOU6dO5dZbb2Xu3LkMGTKEadOmbTHWus7Zrl07FixYwLBhwzjjjDM444wzWLVqFR06dGDKlCn07duXO++8kx/96Ee0adOGzp078+c//5lLLrmEDz/8kL/+9a9ceOGF3HfffVx00UX07dsXSGpPp5122maxTJkyhcmTJ7NhwwZ69+7NbbfdRocOHTa7xuOPP87ixYv59re/zYYNG9i0aRN/+MMf6NOnDx07dmTdunWMHj2adevWMXjwYC688EKWLl1Kx44dOe+883j11Vfz5qV2nidOnFgd29q1a1m4cCGDBg2q3nbXXXcxatQodt11V2bOnMkPf/jDgn8bt99+O0OHDmXUqFHV24YPH17wuEL69etXMM0zzzxD79692XPPPQE49thj+dOf/kS/fv14+OGHuf322wE48cQTueyyyzjttNPo0KEDPXv25JlnnmH//fdvVIyuEVir8fHHH/OXv/yF0aNHA9C3b1+eeOIJFixYwIQJE2p8WFRUVDBr1ixeeOEFZs2axYoVK3jrrbe49NJLefLJJ/nrX/9a/Y0N4Lvf/S4nnngiCxcu5Jvf/GaNb6fvvfcec+fO5Re/+AWjR4/m+9//PosXL+aFF16goqKiOt2IESMoKytjyJAhBc9ZWVnJnDlzmDhxIqeeeiqTJk1i/vz5XH311Zx+ejJVyIQJE3jooYd4/vnnmT17Nttttx0TJkxgzJgxVFRUMGbMGBYtWsTgwYMLvnZHHnkkzz77LM8//zz9+vWr/lZa+xoAN954I2effTYVFRXMmzeP7t271zjX7Nmzq2tpY8aMqbFvS3mpnedc8+bNY+DAgTW2zZgxg7FjxzJ27FhmzJhRMH9A5tdi7dq11U14tX9y/ybqY+XKleyxxx7V6927d2flypWsXr2aLl26sO2229bYXqW8vJwnnniiQdfM5RqBlUx9vrk3pQ8//JCysjJWrlxJv379+PKXvwzAmjVrOPHEE3nllVeQxMaNG6uPOfjgg+ncuTMA/fv354033uDdd99l+PDhdOvWDYAxY8bw8ssvAzB37lzuuit50P6EE07g/PPPrz7XqFGjkMTee+/Nrrvuyt57J/M8DRgwgOXLl1NWVgYkTUM777xz9XF1nfPoo4+mTZs2rFu3jjlz5nD00UdX7/vnP/8JwLBhwxg3bhzHHHMMRx55ZKNew0WLFnHxxRfz/vvvs27dOg455JAtXmPo0KH85Cc/obKykiOPPJI+ffpkukZdecnNc21vvfVW9XsC8Pe//51XXnmFz3/+80iibdu2LFq0iIEDB+a9o6a+d9l06tSpRgHenHbZZRdefPHFRp+nqDUCSYdKeknSMkkX5Nm/vaRZ6f6nJfUsZjzWOlV9+3zjjTeIiOo+gv/+7/9mxIgRLFq0iHvuuafG089VTT6QNJfUty0+V9W5ttlmmxrn3WabbRp83h122AGATZs20aVLl+q+hYqKCpYuXQok38x//OMfs2LFCgYPHszq1as3O8+AAQOYP39+weuNGzeO6667jhdeeIFLL720+rXKd43jjjuu+lv/yJEjefjhhzPlqa685Oa5tvbt29d47+644w7ee+89evXqRc+ePVm+fHl1raBr166899571Wn/8Y9/VBe+WV+LYtQIdt99d1asWFG9XllZye67707Xrl15//33q/9OqrZXqWrWbKyiFQTp8NXXA18F+gNjJfWvlexk4L2I6A38AriyWPGYdejQgWuvvZZrrrmGjz76iDVr1lT/U9XVVl9lyJAhPPbYY6xevZqNGzdy5513Vu874IADmDlzJgDTp0/nwAMPbHS8Wc6544470qtXr+pYIoLnn38egFdffZUhQ4YwYcIEunXrxooVK+jUqRNr166tPn78+PFcccUV1TWbTZs2ceONN252nbVr17LbbruxceNGpk+fXr093zVee+019txzT8466yyOOOKI6j6ZQurKS1369evHsmXLqtdnzJjBgw8+yPLly1m+fDnz58+vfh2HDx/OrFmzqu+8mTZtGiNGjADguOOOY86cOdx3333V53r88cdZtGhRjetV1Qjy/fTvX/sjLpvPfvazvPLKK7z++uts2LCBmTNnMnr0aCQxYsSI6judbrnlFo444ojq415++eXNmsUaopg1gv2BZRHxWkRsAGYCR9RKcwRwS7r8e+Bg+ekjK6J9992XffbZhxkzZnD++edz4YUXsu+++2b6Zr7bbrtx2WWXMXToUIYNG1ajE3DSpEncfPPN7LPPPtx222386le/anSsWc85ffp0brrpJgYNGsSAAQP405/+BCQf8nvvvTcDBw7kgAMOYNCgQYwYMYIlS5ZQVlbGrFmz2GefffjlL3/J2LFj6devHwMHDsx7l83ll1/OkCFDGDZsWHXH8paucccddzBw4EDKyspYtGhRve4I2lJe6tK3b1/WrFnD2rVrWb58OW+88UaN20Z79epF586defrppzn88MM58MADGTx4MGVlZTz55JNceWXy/bN9+/bce++9TJo0iT59+tC/f39uuOGGGs1ODXH33XfTvXt35s6dy2GHHVbdrPbmm28ycuRIALbddluuu+46DjnkEPr168cxxxzDgAFJU+qVV17JxIkT6d27N6tXr65xm+yTTz5Z3dTZGIoozpBBko4CDo2IU9L1E4AhEXFmTppFaZrKdP3VNM27tc51KnAqQI8ePQa/8cYb9Y7nfxa/zR8rVjLxmDLatd28ndGKY+nSpZnumjBrjF/84hd06tSJU045pblDKZkFCxYwceJEbrvtts325fu/kzQ/IsrznatF3DUUEZMjojwiyhtaOn9lwH9wwzcHuxAw2wqddtppNfpfWoN3332Xyy+/vEnOVcy7hlYCe+Ssd0+35UtTKWlboDPJvAdmZpm1a9eOE044obnDKKmmaBKqUswawbNAH0m9JG0HHAvMrpVmNnBiunwU8HAUq63Kmo3fUrPSacj/W9EKgoj4CDgTeAhYCtwREYslTZA0Ok12E9BV0jLgHGCzW0ytZWvXrh2rV692YWBWAlXzEbRr165exxWts7hYysvLo66Bm+zfi2coMyutLc1QVldnsZ8stqJq27ZtvWZKMrPSaxF3DZmZWfG4IDAza+VcEJiZtXItrrNY0iqg/o8WJ3YG3i2YauviPLcOznPr0Jg8fyoi8j6R2+IKgsaQNG9LveZbK+e5dXCeW4di5dlNQ2ZmrZwLAjOzVq61FQSTmzuAZuA8tw7Oc+tQlDy3qj4CMzPbXGurEZiZWS0uCMzMWrmtsiCQdKiklyQtk7TZiKaStpc0K93/tKSezRBmk8qQ53MkLZG0UNJfJH2qOeJsSoXynJPuG5JCUou/1TBLniUdk77XiyXdXuoYm1qGv+0ekh6RtCD9+x7ZHHE2FUlTJb2TzuCYb78kXZu+Hgsl7dfoi0bEVvUDtAFeBfYEtgOeB/rXSnM6cGO6fCwwq7njLkGeRwAd0uXTWkOe03SdgMeBp4Dy5o67BO9zH2AB8Il0fZfmjrsEeZ4MnJYu9weWN3fcjczzQcB+wKIt7B8JPAAI+BzwdGOvuTXWCPYHlkXEaxGxAZgJHFErzRHALeny74GDJamEMTa1gnmOiEci4oN09SmSGeNasizvM8DlwJXA1jAOdpY8/ydwfUS8BxAR75Q4xqaWJc8B7JgudwbeLGF8TS4iHgf+UUeSI4BbI/EU0EXSbo255tZYEOwOrMhZr0y35U0TyQQ6a4CuJYmuOLLkOdfJJN8oWrKCeU6rzHtExH2lDKyIsrzPnwE+I+lJSU9JOrRk0RVHljxfBhwvqRK4H/huaUJrNvX9fy/I8xG0MpKOB8qBLzR3LMUkaRtgIjCumUMptW1JmoeGk9T6Hpe0d0S835xBFdlYYFpEXCNpKHCbpIERsam5A2sptsYawUpgj5z17um2vGkkbUtSnVxdkuiKI0uekfQl4CJgdET8s0SxFUuhPHcCBgKPSlpO0pY6u4V3GGd5nyuB2RGxMSJeB14mKRhaqix5Phm4AyAi5gLtSAZn21pl+n+vj62xIHgW6COpl6TtSDqDZ9dKMxs4MV0+Cng40l6YFqpgniXtC/yGpBBo6e3GUCDPEbEmInaOiJ4R0ZOkX2R0RLTkeU6z/G3/kaQ2gKSdSZqKXithjE0tS57/BhwMIKkfSUGwqqRRltZs4Fvp3UOfA9ZExFuNOeFW1zQUER9JOhN4iOSOg6kRsVjSBGBeRMwGbiKpPi4j6ZQ5tvkibryMeb4K6AjcmfaL/y0iRjdb0I2UMc9blYx5fgj4iqQlwMfA+IhosbXdjHk+F5gi6fskHcfjWvIXO0kzSArzndN+j0uBtgARcSNJP8hIYBnwAfDtRl+zBb9eZmbWBLbGpiEzM6sHFwRmZq2cCwIzs1bOBYGZWSvngsDMrJVzQdAKSPpYUkXOT8860q5rgutNk/R6eq3n0qc963uO30rqny7/sNa+OY2NMT1P1euySNI9kroUSF/WkJEtJe0m6d50ebikNel1l0q6tAHnG101Cqekr1W9Tun6hPTBwUZJ38OjCqR5tD4P6KV5vzdDuryjb0q6WtIXs17PsnNB0Dp8GBFlOT/LS3DN8RFRBlxA8iBbvUTEKRGxJF39Ya19BzQ+POBfr8tAkudJziiQvozk/u36OgeYkrP+RPralJOMkVOvYYQjYnZE/Cxd/RrJiJtV+y6JiD83IMZ/J9OAfGMkTSL5e7Im5oKgFZLUUcmcBM9JekHSZqN2pt9iH8/5xnxguv0rkuamx94pqWOByz0O9E6PPSc91yJJ30u37SDpPknPp9vHpNsflVQu6WdA+zSO6em+denvmZIOy4l5mqSjJLWRdJWkZ5WM1/5fGV6WuaQDd0naP83jAklzJO2VPtU6ARiTxjImjX2qpGfStPlGPwX4BvBg7Y0R8X/AfKB3Wtt4Ko33bkmfSGM5S/+aR2Jmum2cpOskHQCMBq5KY/p0zmtwqKQ7c16b6m/j9X0PJV2SvpaLJE2WaozUe0LO38j+afqsr0teWxp9MyLeALpK+o/6nM8yaI7xtv1T2h+SJ0wr0p+7SZ4o3zHdtzPJE4pVDxeuS3+fC1yULrchGbtnZ5IP9h3S7T8ALslzvWnAUeny0cDTwGDgBWAHkiecFwP7knxITsk5tnP6+1HS+QOqYspJUxXj14Fb0uXtSEZkbA+cClycbt8emAf0yhPnupz83Qkcmq7vCGybLn8J+EO6PA64Luf4K4Dj0+UuJOP67FDrGr2A+Tnrw4F70+WuwHJgALAQ+EK6fQLwy3T5TWD7qmvUjiP3tc5dT9/jv+W8V78Gjm/ge7hTzvbbgFE579GUdPkg0vHzt/S61Mp7OfDbOv5me5JnPH6SmtU3mvt/amv72eqGmLC8PoykKQIASW2BKyQdBGwi+Sa8K/B2zjHPAlPTtH+MiApJXyBphngy/VK4Hck36XyuknQxyZgvJ5OMBXN3JN+CkXQXcCDJN+VrJF1J8iHxRD3y9QDwK0nbkzQlPB4RH0r6CrBPTht3Z5KB116vdXx7SRVp/pcC/y8n/S2S+pAMWdB2C9f/CjBa0nnpejugR3quKrux+bg3B0paQPLa/4xkoLguEfFYuv8WkoIJkgJiuqQ/kowjlEkkQzM8CIyS9HvgMOB8klFns76HVUZIOh/oAOxEUojfk+6bkV7vcUk7Kuln2dLrkhvfPOCUrPnJ8Q7wyQYcZ3VwQdA6fRPoBgyOiI1KRudsl5sg/cc+iOQDZJqkicB7wP+LiLEZrjE+In5ftSLp4HyJIuLltI18JPBjSX+JiAlZMhER6yU9ChwCjCGZtASSmZu+GxEPFTjFhxFRJqkDyVg2ZwDXkkxm80hEfF1Jx/qjWzheJN9OX6rrGtR6bUn6CA6vPonUuY7jDyP5tj0KuEjS3nWkrW0mcCZJM8u8iFibNutkfQ+R1A64gaR2tkLSZdTMT+0xaoItvC6Sdq1H7FvSjuQ1tSbkPoLWqTPwTloIjAA2m79YyZzGf4+IKcBvSabOewoYJqmqzX8HSZ/JeM0ngK9J6iBpB5JmnSckfRL4ICJ+RzIwXr6O041pzSSfWSSDblXVLiD5UD+t6hhJn0mvmVckM7edBZyrfw1LXjWs77icpGtJmsiqPAR8t6rNXMkIr7W9TNLMsUURsQZ4T2k/DHAC8JiSORX2iIhHSJpwOpM0q+WqHVOux0hez//kX4Vkfd/Dqg/9d9O+hNp3ElX16XyeZBTMNWR7XRrqM0DeuXyt4VwQtE7TgXJJLwDfAl7Mk2Y48HzahDEG+FVErCL5YJwhaSFJk0LfLBeMiOdI2p2fIekz+G1ELAD2Bp5Jm2guBX6c5/DJwEKlncW1/A9Jc8efI5nKEJKCawnwnJJbEH9DgdpvGstCkklOfg78NM177nGPAP2rOotJag5t09gWp+u1z/t/wKtVH7x1OJGkOW0hyd1JE0j6Ln6Xvk8LgGtj8wlmZgLj007ZT9e69sfAvcBX09/U9z1MrzeF5MP3IZImw1zr09fpRpImQMjwuii5EeC3+a6pZPTNucBekiolnZxub0ty40FLHkr835JHHzUrMklfJ2mGu7i5Y2nJ0tdxv4j47+aOZWvjPgKzIouIuyW15Dmx/11sC1zT3EFsjVwjMDNr5dxHYGbWyrkgMDNr5VwQmJm1ci4IzMxaORcEZmat3P8H7EAthu6jxFAAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "y_train_rfc_pred_bowandtfidf = rfc3.predict(x_train_bowandtfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_train, y_train_rfc_pred_bowandtfidf)))\n",
        "print('Precision Score:', (precision_score(y_train, y_train_rfc_pred_bowandtfidf)))\n",
        "print('Recall Score:', (recall_score(y_train,y_train_rfc_pred_bowandtfidf)))\n",
        "print('F1 Score:', (f1_score(y_train, y_train_rfc_pred_bowandtfidf)))\n",
        "\n",
        "y_score_rf_bowandtfidf = rfc3.predict_proba(x_train_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_train, y_score_rf_bowandtfidf[:,1])))\n",
        "print(plot_roc_curve(rfc3, x_train_bowandtfidf, y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 136,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "88yNNUolATRf",
        "outputId": "99a05522-f228-44f1-ef88-a00e646b801f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9865771812080537\n",
            "Precision Score: 0.9795918367346939\n",
            "Recall Score: 0.676056338028169\n",
            "F1 Score: 0.8\n",
            "ROC Score: 0.8504802841510332\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A125C0B700>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAwQklEQVR4nO3deXxU5dn/8c/FlgAJoBAQQQQVZZUoqYhUC7W4VbB1o7hUXOpT96cq/rRatdhaW5e2otZCRRARUFtbXKpPtYoKqASJGMAFKRbUloCIQQgkcP3+OCfjJJlkTkhmQpLv+/XKK3P268wk9zXnvs+5b3N3RESk+WrR0AGIiEjDUiIQEWnmlAhERJo5JQIRkWZOiUBEpJlr1dAB1FaXLl28d+/eDR2GiEijsmTJkg3unpNoWaNLBL179yY/P7+hwxARaVTM7OPqlqlqSESkmVMiEBFp5pQIRESaOSUCEZFmTolARKSZS1kiMLNpZrbezAqrWW5mdq+ZrTKzZWZ2eKpiERGR6qXyimA6cEINy08E+oY/FwN/SGEsIiJSjZQlAnd/Ffi8hlVOAR7xwBtAJzPrnqp4REQaq7Kdu/jlsyv49IttKdl/Q7YR9ADWxk2vC+dVYWYXm1m+meUXFRWlJTgRkT3Brl3OdU8uY+pr/+Ll99en5BiNorHY3ae4e5675+XkJHxCWkSkyXF3bvzru/xl6SdcPfpgzh62f0qO05CJ4BNgv7jpnuE8EZFmz935+dMrmP3WWi4deSBXfPuglB2rIRPBPOCH4d1DRwKb3f2zBoxHRGSP4O7c8fx7TF+4hgtG9GHi8YdgZik7Xso6nTOz2cBIoIuZrQNuAVoDuPuDwHPAScAqYCtwfqpiERFpTH734of8cf5qzjmyFz87uX9KkwCkMBG4+/gkyx24LFXHFxFpjB54ZRW/f+lDzhjak0ljB6U8CUAjaSwWEWkOHnr9X/zm+fcZO2Rf7jjtUFq0SH0SACUCEZE9wqw3P+a2Z1Zw/MBu3H3mEFqmKQmAEoGISIN7csk6bnyqkG/368rk8YfTumV6i2YlAhGRBjTvnU+57sl3+OZBXXjg7MNp0yr9xbISgYhIA3m+8D/8ZG4BefvvzZQfDiWzdcsGiUOJQESkAbz83nqumP02g3t0ZNr536Bdm4YbQj7Skc2sBTAE2BfYBhS6e2o6vRARaeIWrNrA/zy6hEP2yWbGBUeQldFwSQCSJAIzOxD4f8B3gA+BIiATONjMtgJ/BGa4+65UByoi0hS89a/PuXDGYvp0bs/MC4bRsW3rhg4p6RXBLwjGCfif8AGwGDPrCpwFnAvMSE14IiJNx9J/b+L8h99i305tefSiYezVvk1DhwQkSQQ1PR0cVg39rr4DEhFpigo/2cx5096ic1YGj110JDnZGQ0dUsxuV0yZ2Wh3/0d9BiMi0lh9tb2M9cXbWf9lCUVbtrP+y+3BdHEJRcXbeWP1RrpmZ/LYj4axT8fMhg63grq0UDwE9KqvQERE9jTuzqatpbHCPL5wX1+8naLwZ/2XJXy1Y2eV7Vu3NHKyMsjpkMlxA/dh4nGH0HOvdg1wJjVL1lg8r7pFQOf6D0dEJPVKd+5iw5bt1Rbu64u3UxR+sy/d6VW2b9+mJV07ZJKTncHAfTsw6pCu5GRn0DU7g64dMuianUnX7Aw6tWudlk7j6irZFcHRwDnAlkrzDTgiJRGJiOymrTvKYgV5UMBXLNzXfxl8s/986w68avnO3u3b0DU7g5zsDA7K6ZKwcM/JzqB9A9/uWd+Snc0bwFZ3n195gZm9n5qQRES+5u5s3lZapXBf/+X2sC6+JFbQb9leVmX7Vi0sVqD33Ksdh++/FzlZFQv3rh0y6Nw+o0G6d9gTJLtr6MQalh1T/+GISHNRtnMXG7/akaBwL4lV15TXwe/YWfVRpXZtWgaFeHYm/fftwDEJCvecrAz2atcmbd05N1ZN6/pGRBpcSenOSoV75btotlNUXMLGrxJXz+zVrnVQmHfI4IAu7cmJL9zDqpmuHTIb/GncpkTvpIgk5e58ua3s68K9hrtoikuqVs+0bGGx6ph9O2aSu19HchIU7jlZzbd6piEpEYg0Yzt3ORu3bE9auK8v3s6OsqrVM21bt4xVwfTbJ5tj+ubE6uNzwmqbrh0y2FvVM3s0JQKRJqikdGdYgJfUeBfNxi3b2ZWgeqZTu9axb/Df6L13rGCPL9y7ZmeQldGqUdweKTWLnAjM7FZ3v7W6aRFJLXfny5IyiuIL82pukfwyQfVMC4MuYeHerUMmg3t0jCvgvy7cc7IzyGjVMP3iS8OozRXBkiTTIrIbdu5yNn4V/yBTgsK9OLiTZnuC6pmMVi1id8v07ZrFiAM7x76554SFe9fsTPZu3yat4+BK4xE5Ebj70zVNi0hF28t2VqiWKaqmcN/41Q52Jqif6ZDZiq4dggbVob32qlAtE/86W9UzUkfJupiYDCSoQQy4+5X1HpHIHszd2RLrXOzrOvjKhfv64u1s3lZaZfsWBp2zMmJ3ywzo3qFCnXtO3NOrDTVsoTQ/ya4I8tMShUgD27XL+Xzrjip17kWVCvei4u1sK63auVibVi1ihfsBOe058oDOFbomKL+TpnNWhqpnZI+T7MniCgPOmFk7d9+a2pBE6s+Osl2xbggqdCgWV7ivLy5hw5bE1TPZma1i39Bz9+uUsN+ZrtmZdGir6hlpvKKOWTycoNvpLKCXmQ0hGLXs0lQGJ1KdLdvLqhTu64tLwobWr6tsNm2tWj1jBp3bZ8S+pffbJzth4Z6TnUHbNqqekaYvamPx74DjgXkA7v6OmamvIalXu3Y5m7buSNigWvme+K0J+n5v07JF7F733p3bc0SfvasU7kHnYm1o1VJPr4qUq81dQ2srXfpW/U8USaB0565qG1SL4gr3ouLtlCWonsnK+Lp6ZnDPThW7JYhraO3YtnH0/S6yp4maCNaa2VGAm1lr4CpgZerCksbgq+1lCQv3ynfSfP7VjoTbd27fJtbHTN9u2QkL95zsDNq10QPwIqkU9T/sx8DvgR7Ap8ALwGWpCkoaTvnQfJXvlllf6Zt7lKH59tu7HUP3j7v/Pa6htXNWG1qrekZkjxApEbj7BuDs2u7czE4gSCAtgT+5+x2VlvcCZgCdwnWud/fnanscSa5s5y42bNlRbeFe26H5Rh6SU6VwL6+eUediIo1L1LuGDiAo0I8keMBsEfATd19dwzYtgfuB0cA6YLGZzXP3FXGr3QQ87u5/MLMBwHNA7905keZq246dFQb1qFy412ZovgNzOics3Jvi0Hwi8rWo/92PERTq3w+nfwDMBobVsM0RwKryZGFmc4BTgPhE4ECH8HVHgmqnZi/R0HyJCvei4u0UJx2ary2H9dor4cNNXdT3u4gQPRG0c/eZcdOPmtnEJNv0ANbGTa+jauK4Ffg/M7sCaA98J9GOzOxi4GKAXr16RQx5z1N5aL7Ed9EkH5ovJzuD/t07cMzBiR5u0tB8IlI7yfoa2jt8+Xczux6YQ/AtfhxBNU5djQemu/vd4UNrM81skLtXKAXdfQowBSAvL6/avo8aSvzQfNUV7uuLt7Pxq+3VDs1X3qBaeWi+8sJdQ/OJSKokK1mWEBT85V8v/ydumQM31LDtJ8B+cdM9w3nxLgROAHD3RWaWCXQB1ieJK+Xih+ar6RbJ9TUMzdclqw1dszPp3jGTIXFD88UX7l2y2qjvdxFpUMn6GupTh30vBvqaWR+CBPAD4KxK6/wbOBaYbmb9gUygqA7HTCp+aL5kt0gm6vs9s3WL2Lf1Q/bJ5uhwaL5Y4a6h+USkkanNCGWDgAEEhTUA7v5Ideu7e5mZXU7wzEFLYJq7LzezSUC+u88DrgGmmtlPCK4wJrgnqjypH+c+9Cavfbgh4bKObVvHGlQ1NJ+INCdRbx+9BRhJkAieA04EXgeqTQQA4TMBz1Wad3Pc6xXAiFpFXAcF//4CgNtOGVhhaL4uWer7XUSar6hXBKcDQ4Cl7n6+mXUDHk1dWKnRsqVx3vD9OXd474YORURkjxH1JvJt4Z08ZWbWgaAxd78k2+xxdu1yVeuIiFQS9Yog38w6AVMJ7iTaQvB0caNRPsSgbsEUEakoal9D5QPQPGhmzwMd3H1Z6sKqf1u2l7HLoUNbJQIRkXjJHig7vKZl7v52/YeUGl+G9/p3bNu6gSMREdmzJPt6fHcNyxz4dj3GklJfbguGLOyQqUQgIhIv2QNlo9IVSKptLk8EuiIQEamg2XQ9WVIaDKKi5wVERCpqNomgnO4eFRGpqNklAhERqShSIrDAOWZ2czjdy8yOSG1oIiKSDlGvCB4AhhOMHwBQTDBimYiINHJRn64a5u6Hm9lSAHffZGZtUhiXiIikSdQrgtJwMHoHMLMcoGpn/SIi0uhETQT3Ak8BXc3slwRdUN+esqhERCRtovY1NMvMlhCMJmbA99x9ZUojExGRtIg6MM29wBx3VwOxiEgTE7VqaAlwk5l9ZGZ3mVleKoMSEZH0iZQI3H2Gu58EfAN4H/i1mX2Y0shERCQtavtk8UFAP2B/4L36D0dERNIt6pPFvwmvACYBhUCeu49JaWQiIpIWUR8o+wgY7u4bUhmMiIikX7IRyvq5+3vAYqCXmfWKX96YRigTEZHEkl0RXA1cTOKRyhrVCGUiIpJYshHKLg5fnujuJfHLzCwzZVGJiEjaRL1raGHEeSIi0sgkayPYB+gBtDWzwwi6lwDoALRLcWwiIpIGydoIjgcmAD2Be+LmFwM/TVFMIiKSRsnaCGYAM8zsNHf/c5piEhGRNEpWNXSOuz8K9Dazqysvd/d7EmwmIiKNSLLG4vbh7ywgO8FPjczsBDN738xWmdn11axzppmtMLPlZvZYLWIXEZF6kKxq6I/h75/XdsfhiGb3A6OBdcBiM5vn7ivi1ukL3ACMCIe/7Frb44iISN3Upq+hDmbW2sxeMrMiMzsnyWZHAKvcfbW77wDmAKdUWudHwP3uvgnA3dfX9gRERKRuoj5HcJy7fwmcDKwh6IV0YpJtegBr46bXhfPiHQwcbGYLzOwNMzsh0Y7M7GIzyzez/KKiooghi4hIFFETQXkV0neBJ9x9cz0dvxXQFxgJjAemmlmnyiu5+xR3z3P3vJycnHo6tIiIQPRE8IyZvQcMBV4ysxygJMk2nwD7xU33DOfFWwfMc/dSd/8X8AFBYhARkTSJOkLZ9cBRBOMQlAJfUbW+v7LFQF8z62NmbYAfAPMqrfNXgqsBzKwLQVXR6qjBi4hI3UUdvL41cA5wjJkBzAcerGkbdy8zs8uBF4CWwDR3X25mk4B8d58XLjvOzFYAO4GJ7r5xt89GRERqLerANH8AWgMPhNPnhvMuqmkjd38OeK7SvJvjXjtBV9dVHlYTEZH0iJoIvuHuQ+Km/2lm76QiIBERSa+ojcU7zezA8gkzO4CgKkdERBq5qFcEE4GXzWw1QVfU+wPnpywqERFJm6SJILxVdDPBk8LlXUC87+7bUxmYiIikR41VQ2Z2EbAcmAwUAL3dfZmSgIhI05HsiuB/gYHuXhS2C8yi6rMAIiLSiCVrLN7h7kUA7r4ayEh9SCIikk7Jrgh6mtm91U27+5WpCUtERNIlWSKo3MPoklQFIiIiDSPKmMUiItKEJbtraKqZDapmWXszu8DMzk5NaCIikg7JqobuB242s8FAIVAEZBJ0Fd0BmEZwJ5GIiDRSyaqGCoAzzSwLyAO6A9uAle7+furDExGRVIvUxYS7bwFeSW0oIiLSEKJ2OiciIk2UEoGISDNXq0RgZu1SFYiIiDSMSInAzI4Kh5N8L5weYmYPJNlMREQagahXBL8Fjgc2Arj7O8AxqQpKRETSJ3LVkLuvrTRLI5SJiDQBUUcoW2tmRwFuZq2Bq4CVqQtLRETSJeoVwY+By4AewCdALnBpimISEZE0inpFcIi7V+hTyMxGAAvqPyQREUmnqFcEkyPOExGRRqbGKwIzGw4cBeSY2dVxizoALVMZmIiIpEeyqqE2QFa4Xnbc/C+B01MVlIiIpE+y3kfnA/PNbLq7f5ymmEREJI2iNhZvNbM7gYEE4xEA4O7fTklUIiKSNlEbi2cRdC/RB/g5sAZYnKKYREQkjaImgs7u/hBQ6u7z3f0CQFcDIiJNQNSqodLw92dm9l3gU2Dv1IQkIiLpFPWK4Bdm1hG4BrgW+BPwv8k2MrMTzOx9M1tlZtfXsN5pZuZmlhcxHhERqSdRh6p8Jny5GRgFsSeLq2VmLYH7gdHAOmCxmc1z9xWV1ssm6LvozdqFLiIi9aHGKwIza2lm483sWjMbFM472cwWAvcl2fcRwCp3X+3uO4A5wCkJ1rsN+DVQUvvwRUSkrpJVDT0EXAR0Bu41s0eBu4DfuPthSbbtAcR3Xb0unBdjZocD+7n7szXtyMwuNrN8M8svKipKclgREamNZFVDecCh7r7LzDKB/wAHuvvGuh7YzFoA9wATkq3r7lOAKQB5eXle12OLiMjXkl0R7HD3XQDuXgKsrkUS+ATYL266ZzivXDYwCHjFzNYARwLz1GAsIpJeya4I+pnZsvC1AQeG0wa4ux9aw7aLgb5m1ocgAfwAOKt8obtvBrqUT5vZK8C17p5f67MQEZHdliwR9N/dHbt7mZldDrxA0FPpNHdfbmaTgHx3n7e7+xYRkfqTrNO5OnU05+7PAc9VmndzNeuOrMuxRERk90QevF5ERJomJQIRkWYuciIws7ZmdkgqgxERkfSLlAjMbAxQADwfTueamRp7RUSagKhXBLcSdBnxBYC7FxCMTSAiIo1c1ERQGt73H09P+IqINAFRxyNYbmZnAS3NrC9wJbAwdWGJiEi6RL0iuIJgvOLtwGME3VH/b4piEhGRNIp6RdDP3W8EbkxlMCIikn5RrwjuNrOVZnZb+bgEIiLSNERKBO4+imBksiLgj2b2rpndlNLIREQkLSI/UObu/3H3e4EfEzxTkLDPIBERaVyiPlDW38xuNbN3gckEdwz1TGlkIiKSFlEbi6cBc4Hj3f3TFMYjIiJpFikRuPvwVAeSanr6TUQksRoTgZk97u5nhlVC8WVplBHK9ixh9NawUYiI7HGSXRFcFf4+OdWBpIuZUoGISLwaG4vd/bPw5aXu/nH8D3Bp6sOrP67KIRGRhKLePjo6wbwT6zOQVHNVDYmIJJSsjeASgm/+B5jZsrhF2cCCVAZW32KJQJlARKSCZG0EjwF/B34FXB83v9jdP09ZVClQXjFkuiYQEakgWSJwd19jZpdVXmBmeze2ZAC6IhARqSzKFcHJwBKCL9XxxagDB6QornrnrsZiEZFEakwE7n5y+LvRD0upNCAikljUvoZGmFn78PU5ZnaPmfVKbWj1S43FIiKJRb199A/AVjMbAlwDfATMTFlUKaTGYhGRiqImgjIPKtlPAe5z9/sJbiFtRFQ5JCKSSNTeR4vN7AbgXOBoM2sBtE5dWPVPVUMiIolFvSIYRzBw/QXu/h+CsQjuTFlUKRB7jkCJQESkgqhDVf4HmAV0NLOTgRJ3fySlkdWzr7uYUCYQEYkX9a6hM4G3gDOAM4E3zez0CNudYGbvm9kqM7s+wfKrzWyFmS0zs5fMbP/ankBt6YpARKSiqG0ENwLfcPf1AGaWA7wIPFndBmbWErifoMO6dcBiM5vn7iviVlsK5Ln71rBfo98QVEPVO/U+KiKSWNQ2ghblSSC0McK2RwCr3H21u+8A5hDcdRTj7i+7+9Zw8g1SOA6yeh8VEUks6hXB82b2AjA7nB4HPJdkmx7A2rjpdcCwGta/kKCDuyrM7GLgYoBevXbvOTY1FouIJBZ1zOKJZnYq8M1w1hR3f6q+gjCzc4A84FvVHH8KMAUgLy+vjnU8ygQiIvGSjUfQF7gLOBB4F7jW3T+JuO9PgP3ipnuG8yof4zsEbRDfcvftEfdda+p0TkQksWT1/NOAZ4DTCHognVyLfS8G+ppZHzNrA/wAmBe/gpkdBvwRGFupDSJlVDUkIlJRsqqhbHefGr5+38zejrpjdy8zs8uBF4CWwDR3X25mk4B8d59H8FBaFvBEOKj8v919bK3PIlI8wW/lARGRipIlgszwW3t5+dk2ftrda0wM7v4clRqV3f3muNffqXXEu6n89lHTJYGISAXJEsFnwD1x0/+Jm3bg26kIKpWUBkREKko2MM2odAWSamorFhFJLOoDZY2eeh8VEUms+SSC8Lc6nRMRqajZJIJyuiIQEakoau+jFo5VfHM43cvMjkhtaPVLD5SJiCQW9YrgAWA4MD6cLiboWbTRUBoQEUksaqdzw9z9cDNbCuDum8KnhRsPNRaLiCQU9YqgNBxfwCE2HsGulEWVAnqgTEQksaiJ4F7gKaCrmf0SeB24PWVRpZDSgIhIRVG7oZ5lZkuAYwnK0u+5+8qURlbP1FYsIpJYpERgZr2ArcDT8fPc/d+pCqy+aWAaEZHEojYWP0tQlhqQCfQB3gcGpiiuevd176PKBCIi8aJWDQ2Onzazw4FLUxJRiumKQESkot16sjjsfrqm8Yf3OK4nCUREEoraRnB13GQL4HDg05RElCIamEZEJLGobQTZca/LCNoM/lz/4aRO7HpAmUBEpIKkiSB8kCzb3a9NQzypE14SqLFYRKSiGtsIzKyVu+8ERqQpnpRTY7GISEXJrgjeImgPKDCzecATwFflC939LymMrV6pqVhEJLGobQSZwEaCMYrLnydwoPEkAjUWi4gklCwRdA3vGCrk6wRQrlF9yS4fj0CdzqVXaWkp69ato6SkpKFDEWkWMjMz6dmzJ61bt468TbJE0BLIIvEX6UaVCMopDaTXunXryM7Opnfv3krCIinm7mzcuJF169bRp0+fyNslSwSfufukuoW2Z2iUWasJKCkpURIQSRMzo3PnzhQVFdVqu2RPFjeZ/17XwDQNRklAJH125/8tWSI4dvdC2fPEeh9tOrlNRKRe1JgI3P3zdAWSaq7bhpqtli1bkpuby6BBgxgzZgxffPFFvex3+vTpXH755fWyr969ezN48GByc3PJzc1l4cKF9bLfygoKCnjuuecqzPv73/9OXl4eAwYM4LDDDuOaa64B4NZbb+Wuu+6qt2MfddRRsdcTJ05k4MCBTJw4kQcffJBHHnmkTvteunQpF154YYV53/ve9zjyyCMrzJswYQJPPvlkhXlZWVmx1x988AEnnXQSffv25fDDD+fMM8/kv//9b51i+/zzzxk9ejR9+/Zl9OjRbNq0KeF61113HQMHDqR///5ceeWVsTJr5MiRHHLIIbG/jfXr1wNw3333MW3atDrFVm63Op1rzFRL0fy0bduWgoICCgsL2Xvvvbn//vsbOqSEXn75ZQoKCigoKKhQaNakrKysVseonAgKCwu5/PLLefTRR1mxYgX5+fkcdNBBtdpnVPHJbcqUKSxbtow777yTH//4x/zwhz+MvJ9E53z77bdz5ZVXxqa/+OILlixZwubNm1m9enWk/ZaUlPDd736XSy65hA8//JC3336bSy+9tNb17ZXdcccdHHvssXz44Ycce+yx3HHHHVXWWbhwIQsWLGDZsmUUFhayePFi5s+fH1s+a9as2N9G165dAbjggguYPHlynWIrF/U5ApE6+/nTy1nx6Zf1us8B+3bgljHRh8UYPnw4y5YtA+Ctt97iqquuoqSkhLZt2/Lwww9zyCGHMH36dObNm8fWrVv56KOP+P73v89vfvMbAB5++GF+9atf0alTJ4YMGUJGRgYAa9as4YILLmDDhg3k5OTw8MMP06tXLyZMmEDbtm1ZunQp69evZ9q0aTzyyCMsWrSIYcOGMX369GpjrWmfmZmZLF26lBEjRnDZZZdx2WWXUVRURLt27Zg6dSr9+vXjiSee4Oc//zktW7akY8eOvPjii9x8881s27aN119/nRtuuIFnn32WG2+8kX79+gHB1dMll1xSJZapU6cyZcoUduzYwUEHHcTMmTNp165dlWO8+uqrLF++nPPPP58dO3awa9cu/vznP9O3b1+ysrLYsmULY8eOZcuWLQwdOpQbbriBlStXkpWVxbXXXstHH32U8Fwqn/M999wTi624uJhly5YxZMiQ2Ly//OUvjBkzhm7dujFnzhx++tOfJv3beOyxxxg+fDhjxoyJzRs5cmTS7ZL529/+xiuvvALAeeedx8iRI/n1r39dYR0zo6SkhB07duDulJaW0q1btxr3265dO3r37s1bb73FEUccUacYm80VgWqGZOfOnbz00kuMHTsWgH79+vHaa6+xdOlSJk2aVKGwKCgoYO7cubz77rvMnTuXtWvX8tlnn3HLLbewYMECXn/9dVasWBFb/4orruC8885j2bJlnH322RW+nW7atIlFixbx29/+lrFjx/KTn/yE5cuX8+6771JQUBBbb9SoUeTm5jJs2LCk+1y3bh0LFy7knnvu4eKLL2by5MksWbKEu+66i0svDYYKmTRpEi+88ALvvPMO8+bNo02bNkyaNIlx48ZRUFDAuHHjKCwsZOjQoUnfu1NPPZXFixfzzjvv0L9/fx566KGExwB48MEHueqqqygoKCA/P5+ePXtW2Ne8efNiV2njxo2rsKy6c6l8zvHy8/MZNGhQhXmzZ89m/PjxjB8/ntmzZyc9PyDye1FcXByrpqn8E/83Ue6///0v3bt3B2CfffZJWNU0fPhwRo0aRffu3enevTvHH388/fv3jy0///zzyc3N5bbbbvu6mhvIy8vjtddei3R+NWk2VwTl4xHoDpaGU5tv7vVp27Zt5Obm8sknn9C/f39Gjx4NwObNmznvvPP48MMPMTNKS0tj2xx77LF07NgRgAEDBvDxxx+zYcMGRo4cSU5ODgDjxo3jgw8+AGDRokX85S/Bg/bnnnsu1113XWxfY8aMwcwYPHgw3bp1Y/DgYJyngQMHsmbNGnJzc4GgaqhLly6x7Wra5xlnnEHLli3ZsmULCxcu5Iwzzogt2759OwAjRoxgwoQJnHnmmZx66ql1eg8LCwu56aab+OKLL9iyZQvHH398tccYPnw4v/zlL1m3bh2nnnoqffv2jXSMms4l/pwr++yzz2KfCQQF74cffsg3v/lNzIzWrVtTWFjIoEGDEv7/17ZMyM7OrpDAa8PMEh5v1apVrFy5knXr1gEwevRoXnvtNY4++mhmzZpFjx49KC4u5rTTTmPmzJmxqrSuXbvy3nvv7VYs8VJ6RWBmJ5jZ+2a2ysyuT7A8w8zmhsvfNLPeqYwHdEXQHJV/+/z4449x91gbwc9+9jNGjRpFYWEhTz/9dIWnn8urfCCoLqltXXy88n21aNGiwn5btGix2/tt3749ALt27aJTp06x+uOCggJWrlwJBN/Mf/GLX7B27VqGDh3Kxo0bq+xn4MCBLFmyJOnxJkyYwH333ce7777LLbfcEnuvEh3jrLPOin3rP+mkk/jnP/8Z6ZxqOpf4c66sbdu2FT67xx9/nE2bNtGnTx969+7NmjVrYlcFnTt3rtBY+/nnn8eSb9T3orZXBN26deOzzz4DgqRVXscf76mnnuLII48kKyuLrKwsTjzxRBYtWgRAjx49gCABnXXWWbz11lux7cqrNesqZYkg7L76fuBEYAAw3swGVFrtQmCTux8E/Bb4NSnieqKs2WvXrh333nsvd999N2VlZWzevDn2T1ZTXX25YcOGMX/+fDZu3EhpaSlPPPFEbNlRRx3FnDlzgKBh7+ijj65zvFH22aFDB/r06ROLxd155513APjoo48YNmwYkyZNIicnh7Vr15KdnU1xcXFs+4kTJ3L77bfHrmx27drFgw8+WOU4xcXFdO/endLSUmbNmhWbn+gYq1ev5oADDuDKK6/klFNOibXJJFPTudSkf//+rFq1KjY9e/Zsnn/+edasWcOaNWtYsmRJ7H0cOXIkc+fOZceOHUDwuY8aNQqAs846i4ULF/Lss8/G9vXqq69SWFhY4XjlVwSJfgYMqFzEwdixY5kxYwYAM2bM4JRTTqmyTq9evZg/fz5lZWWUlpYyf/58+vfvT1lZGRs2bACC7lqeeeaZCtVgH3zwQZVqsd2RyiuCI4BV7r7a3XcAc4DK78ApwIzw9ZPAsZaiupvYcwS6JGjWDjvsMA499FBmz57Nddddxw033MBhhx0W6Zt59+7dufXWWxk+fDgjRoyoUIc7efJkHn74YQ499FBmzpzJ73//+zrHGnWfs2bN4qGHHmLIkCEMHDiQv/3tb0BQyA8ePJhBgwZx1FFHMWTIEEaNGsWKFSvIzc1l7ty5HHroofzud79j/Pjx9O/fn0GDBiW8y+a2225j2LBhjBgxItawXN0xHn/8cQYNGkRubi6FhYW1uiOounOpSb9+/di8eTPFxcWsWbOGjz/+uMJto3369KFjx468+eabnHzyyRx99NEMHTqU3NxcFixYEGu4bdu2Lc888wyTJ0+mb9++DBgwgAceeKBCtdPuuP766/nHP/5B3759efHFF7n++qByJD8/n4suugiA008/nQMPPJDBgwczZMgQhgwZwpgxY9i+fTvHH388hx56KLm5ufTo0YMf/ehHsX0vWLAgVtVZF+Yp+qpsZqcDJ7j7ReH0ucAwd788bp3CcJ114fRH4TobKu3rYuBigF69eg39+OOPax3P/y3/D38r+JR7xg0ho1XVekZJjZUrV1YoMEVS4be//S3Z2dmxgrU5WLp0Kffccw8zZ86ssizR/52ZLXH3vET7ahR3Dbn7FHfPc/e83c3Oxw3ch/vPPlxJQKQJuuSSSyq0vzQHGzZs4LbbbquXfaXyrqFPgP3ipnuG8xKts87MWgEdCcY9EBGJLDMzk3PPPbehw0ir+qgSKpfKK4LFQF8z62NmbYAfAPMqrTMPOC98fTrwT09VXZU0GH2kIumzO/9vKUsE7l4GXA68AKwEHnf35WY2yczGhqs9BHQ2s1XA1UCVW0ylccvMzGTjxo1KBiJpUD4eQWZmZq22S1ljcark5eV5fn5+Q4chEWmEMpH0qm6Espoai5vNk8XSMFq3bl2rkZJEJP0axV1DIiKSOkoEIiLNnBKBiEgz1+gai82sCKj9o8WBLsCGpGs1LTrn5kHn3DzU5Zz3d/eET+Q2ukRQF2aWX12reVOlc24edM7NQ6rOWVVDIiLNnBKBiEgz19wSwZSGDqAB6JybB51z85CSc25WbQQiIlJVc7siEBGRSpQIRESauSaZCMzsBDN738xWmVmVHk3NLMPM5obL3zSz3g0QZr2KcM5Xm9kKM1tmZi+Z2f4NEWd9SnbOceudZmZuZo3+VsMo52xmZ4af9XIzeyzdMda3CH/bvczsZTNbGv59n9QQcdYXM5tmZuvDERwTLTczuzd8P5aZ2eF1Pqi7N6kfoCXwEXAA0AZ4BxhQaZ1LgQfD1z8A5jZ03Gk451FAu/D1Jc3hnMP1soFXgTeAvIaOOw2fc19gKbBXON21oeNOwzlPAS4JXw8A1jR03HU852OAw4HCapafBPwdMOBI4M26HrMpXhEcAaxy99XuvgOYA5xSaZ1TgBnh6yeBY80a9bD2Sc/Z3V92963h5BsEI8Y1ZlE+Z4DbgF8DTaEf7Cjn/CPgfnffBODu69McY32Lcs4OdAhfdwQ+TWN89c7dXwU+r2GVU4BHPPAG0MnMutflmE0xEfQA1sZNrwvnJVzHgwF0NgOd0xJdakQ553gXEnyjaMySnnN4ybyfuz+bzsBSKMrnfDBwsJktMLM3zOyEtEWXGlHO+VbgHDNbBzwHXJGe0BpMbf/fk9J4BM2MmZ0D5AHfauhYUsnMWgD3ABMaOJR0a0VQPTSS4KrvVTMb7O5fNGRQKTYemO7ud5vZcGCmmQ1y910NHVhj0RSvCD4B9oub7hnOS7iOmbUiuJzcmJboUiPKOWNm3wFuBMa6+/Y0xZYqyc45GxgEvGJmawjqUuc18gbjKJ/zOmCeu5e6+7+ADwgSQ2MV5ZwvBB4HcPdFQCZB52xNVaT/99poiolgMdDXzPqYWRuCxuB5ldaZB5wXvj4d+KeHrTCNVNJzNrPDgD8SJIHGXm8MSc7Z3Te7exd37+3uvQnaRca6e2Me5zTK3/ZfCa4GMLMuBFVFq9MYY32Lcs7/Bo4FMLP+BImgKK1Rptc84Ifh3UNHApvd/bO67LDJVQ25e5mZXQ68QHDHwTR3X25mk4B8d58HPERw+biKoFHmBw0Xcd1FPOc7gSzgibBd/N/uPrbBgq6jiOfcpEQ85xeA48xsBbATmOjujfZqN+I5XwNMNbOfEDQcT2jMX+zMbDZBMu8StnvcArQGcPcHCdpBTgJWAVuB8+t8zEb8fomISD1oilVDIiJSC0oEIiLNnBKBiEgzp0QgItLMKRGIiDRzSgTNgJntNLOCuJ/eNay7pR6ON93M/hUe6+3wac/a7uNPZjYgfP3TSssW1jXGcD/l70uhmT1tZp2SrJ+7Oz1bmll3M3smfD3SzDaHx11pZrfsxv7GlvfCaWbfK3+fwulJ4YODdRJ+hqcnWeeV2jygF577MxHWS9j7ppndZWbfjno8iU6JoHnY5u65cT9r0nDMie6eC1xP8CBbrbj7Re6+Ipz8aaVlR9U9PODr92UQwfMklyVZP5fg/u3auhqYGjf9Wvje5BH0kVOrboTdfZ673xFOfo+gx83yZTe7+4u7EeOeZDqQqI+kyQR/T1LPlAiaITPLsmBMgrfN7F0zq9JrZ/gt9tW4b8xHh/OPM7NF4bZPmFlWksO9ChwUbnt1uK9CM/vfcF57M3vWzN4J548L579iZnlmdgfQNoxjVrhsS/h7jpl9Ny7m6WZ2upm1NLM7zWyxBf21/0+Et2URYcddZnZEeI5LzWyhmR0SPtU6CRgXxjIujH2amb0Vrpuo91OA04DnK89096+AJcBB4dXGG2G8T5nZXmEsV9rX40jMCedNMLP7zOwoYCxwZxjTgXHvwQlm9kTcexP7Nl7bz9DMbg7fy0Izm2JWoafec+P+Ro4I14/6viRUXe+b7v4x0NnM9qnN/iSChuhvWz/p/SF4wrQg/HmK4InyDuGyLgRPKJY/XLgl/H0NcGP4uiVB3z1dCAr29uH8/wfcnOB404HTw9dnAG8CQ4F3gfYETzgvBw4jKCSnxm3bMfz9CuH4AeUxxa1THuP3gRnh6zYEPTK2BS4GbgrnZwD5QJ8EcW6JO78ngBPC6Q5Aq/D1d4A/h68nAPfFbX87cE74uhNBvz7tKx2jD7Akbnok8Ez4ujOwBhgILAO+Fc6fBPwufP0pkFF+jMpxxL/X8dPhZ/zvuM/qD8A5u/kZ7h03fyYwJu4zmhq+Poaw//zq3pdK554H/KmGv9neJOiPn+DK6rSG/p9qaj9NrosJSWibB1URAJhZa+B2MzsG2EXwTbgb8J+4bRYD08J1/+ruBWb2LYJqiAXhl8I2BN+kE7nTzG4i6PPlQoK+YJ7y4FswZvYX4GiCb8p3m9mvCQqJ12pxXn8Hfm9mGQRVCa+6+zYzOw44NK6OuyNBx2v/qrR9WzMrCM9/JfCPuPVnmFlfgi4LWldz/OOAsWZ2bTidCfQK91WuO1X7vTnazJYSvPd3EHQU18nd54fLZxAkJggSxCwz+ytBP0KReNA1w/PAGDN7EvgucB1Br7NRP8Nyo8zsOqAdsDdBEn86XDY7PN6rZtbBgnaW6t6X+PjygYuink+c9cC+u7Gd1ECJoHk6G8gBhrp7qQW9c2bGrxD+Yx9DUIBMN7N7gE3AP9x9fIRjTHT3J8snzOzYRCu5+wdhHflJwC/M7CV3nxTlJNy9xMxeAY4HxhEMWgLByE1XuPsLSXaxzd1zzawdQV82lwH3Egxm87K7f9+ChvVXqtneCL6dvl/TMaj03hK0EZwc24lZxxq2/y7Bt+0xwI1mNriGdSubA1xOUM2S7+7FYbVO1M8QM8sEHiC4OltrZrdS8Xwq91HjVPO+mFm3WsRenUyC91TqkdoImqeOwPowCYwCqoxfbMGYxv9196nAnwiGznsDGGFm5XX+7c3s4IjHfA34npm1M7P2BNU6r5nZvsBWd3+UoGO8RA2npeGVSSJzCTrdKr+6gKBQv6R8GzM7ODxmQh6M3HYlcI193S15ebe+E+JWLSaoIiv3AnBFeZ25BT28VvYBQTVHtdx9M7DJwnYY4FxgvgVjKuzn7i8TVOF0JKhWi1c5pnjzCd7PH/F1kqztZ1he6G8I2xIq30lU3qbzTYJeMDcT7X3ZXQcDCcfyld2nRNA8zQLyzOxd4IfAewnWGQm8E1ZhjAN+7+5FBAXjbDNbRlCl0C/KAd39bYJ657cI2gz+5O5LgcHAW2EVzS3ALxJsPgVYZmFjcSX/R1Dd8aIHQxlCkLhWAG9bcAviH0ly9RvGsoxgkJPfAL8Kzz1+u5eBAeWNxQRXDq3D2JaH05X3+xXwUXnBW4PzCKrTlhHcnTSJoO3i0fBzWgrc61UHmJkDTAwbZQ+sdOydwDPAieFvavsZhsebSlD4vkBQZRivJHyfHiSoAoQI74sFNwL8KdExLeh9cxFwiJmtM7MLw/mtCW48aMxdie+R1PuoSIqZ2fcJquFuauhYGrPwfTzc3X/W0LE0NWojEEkxd3/KzBrzmNh7ilbA3Q0dRFOkKwIRkWZObQQiIs2cEoGISDOnRCAi0swpEYiINHNKBCIizdz/Byjzi/CXIW3MAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "X_test_bowandtfidf =  np.hstack((X_test_pred[:,[0,1 ,2,3, 4]], X_test_pred_tfidf[:,[0,1,2,3,4]]))\n",
        "\n",
        "#Test results\n",
        "y_test_rfc_pred_bowandtfidf = rfc3.predict(X_test_bowandtfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_test, y_test_rfc_pred_bowandtfidf)))\n",
        "print('Precision Score:', (precision_score(y_test, y_test_rfc_pred_bowandtfidf)))\n",
        "print('Recall Score:', (recall_score(y_test,y_test_rfc_pred_bowandtfidf)))\n",
        "print('F1 Score:', (f1_score(y_test, y_test_rfc_pred_bowandtfidf)))\n",
        "\n",
        "y_score_rf_bowandtfidf = rfc3.predict_proba(X_test_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_test, y_score_rf_bowandtfidf[:,1])))\n",
        "print(plot_roc_curve(rfc3, X_test_bowandtfidf, y_test))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uCPoH8LgMEtk"
      },
      "source": [
        "Accuracy Score: 0.9848993288590604\n",
        "Precision Score: 0.9523809523809523\n",
        "Recall Score: 0.7142857142857143\n",
        "F1 Score: 0.8163265306122449\n",
        "ROC Score: 0.9088070087189806"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 137,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "id": "lOU0yVJVATTo",
        "outputId": "5ed8a42d-a490-4454-ff62-3236f200e808"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'Mean decrease in impurity')"
            ]
          },
          "execution_count": 137,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAELCAYAAADDZxFQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAf9klEQVR4nO3dfbxcVX3v8c+XIAHk2USEhJCI0QraYo2gQoUiSBAhtsXycKmIKMXCRV+W1iCWh6g00Fu1fQGvK0qs4AMKKoYSi1iEW6VKAkQgICXECIkIkWcBA0m+94+9D06Gfebsk5w9M8n5vl+vec3stffa6zdzkvnNWms/yDYRERHtNul1ABER0Z+SICIiolISREREVEqCiIiISkkQERFRKQkiIiIqJUHEBknSxyV9sddxjCb5zEcf5TyI0UfSUmBHYHVL8att/2o99/kB2z9Yv+g2PJLOBl5l+9hex7KhkmRgBbCz7VVl2UuA5cB42yrLbgDeDDwPGLgXuAL4rO2V5TZnk7/HiEgPYvQ6zPZWLY91Tg4jQdKmvWx/XW2ocfepx4BDWpYPKcvanWJ7a2An4G+Bo4B5ktR8iKNLEkS8QNK2ki6R9KCk5ZI+JWlMuW43SddLekTSbyR9VdJ25brLgEnA1ZJ+K+nvJe0vaVnb/pdKOrB8fbakKyV9RdKTwPs6tV8R69mSvlK+nizJko6X9ICkxySdJOlNkm6X9LikC1rqvk/SjyVdIOkJST+X9PaW9TtLmivpUUmLJX2wrd3WuE8CPg4cWb73n5XbHS/pbklPSVoi6a9b9rG/pGWS/lbSw+X7Pb5l/RaS/lnSL8v4fiRpi3LdmyXdVL6nn0nav+19LSnb/IWk/zXIZ/dvkj7VHk/L8sfKz/8pSfcMfDaDfObHSbq//DdxRtt7+HL5t7i7/Dex1r+HCpcB721Zfi9w6WAb237a9g3A4cBbgEOH2H8MUxJEtPo3YBXwKuANwDuAD5TrBPwjsDPwWmAX4GwA238F3M/veyXn12xvBnAlsB3w1SHar2NvYCpwJPA54AzgQGAP4C8l7de27X3AOOAs4NuSdijXXQ4sK9/rEcC5kg4YJO5LgHOBb5Tv/Y/KbR4G3gVsAxwPfFbSH7fs4xXAtsAE4ATgQknbl+v+D/BG4K3ADsDfA2skTQCuAT5Vlp8GfEvSeEkvBf4VOKT8df1WYOEwPjsAJL0GOAV4U7mfg4GlHarsC7wGeDtwpqTXluVnAZOBVwIHAXWGe64C3iZpu/Kz+BPgu0NVsn0/sKDcPkZQEsTodVX5K/RxSVdJ2hF4J/CR8pfZw8BnKbrv2F5s+zrbK22vAD4D7Df47mv5b9tX2V5D8UU6aPs1fdL272x/H3ga+Lrth20vB/6LIukMeBj4nO3nbX8DuAc4VNIuwD7Ax8p9LQS+yNq/bF+I2/azVYHYvsb2fS7cCHyftb/Angdmle3PA34LvEbSJsD7gQ/bXm57te2byvH1Y4F5tueVbV9H8cX4znKfa4DXSdrC9oO2Fw3jsxuwGhgL7C7pJbaX2r6vw/bn2H7W9s+AnwEDCfIvgXNtP2Z7GUXyGsrvgKspEvyRwNyyrI5fUSTNGEFJEKPXu21vVz7eDewKvAR4cCBxAJ8HXg4gaUdJl5dDD08CX6H49b0+Hmh53bH9mh5qef1sxfJWLcvLvfYRGr+k6DHsDDxq+6m2dRMGibuSpEMk/aQcpnqc4ku89fN6ZGAytvRMGd84YHOK3k27XYH3tCT2xyl+we9k+2mKL9WTKD7DayT9wVBxtrO9GPgIRe/w4fJvvnOHKr+ueA9QfI6tn9OQn1npUopk3HF4qcIE4NFhbB81JEHEgAeAlcC4lsSxje09yvXnUhw18nrb21D8mm2dFGw/HO5pYMuBhXIuYXzbNq11hmp/pE2Q1prUnETxK/RXwA6Stm5bt3yQuF+0LGks8C2KoaIdbW8HzGPtz2swv6H41bxbxboHgMtaPp/tbL/U9mwA29faPohi8vbnwBcGaWOtvw3FcNfv34z9Ndv7UiQkA+fViLvdg8DEluVdatb7L4r4dwR+VKdC2et7Y1k3RlASRABg+0GKYZB/lrSNpE1UTEwPDCNtTTEM8kQ5Fv53bbt4iGK8ecD/AJtLOlTF4YqfoBi6WNf2R9rLgVMlvUTSeyjmVebZfgC4CfhHSZtL+kOKOYKvdNjXQ8DkcngIYDOK97oCWCXpEIr5lCGVw21zgM+omCwfI+ktZdL5CnCYpIPL8s3LCeaJZQ9vRjkXsZLib7VmkGYWAu+UtIOkV1D0GIBiDkLSAWV7v6PoeQ22n06+CZwuafvy38spNd+/gcOAw9t6eC8iacvy38d3gZspknCMoCSIaPVeii+3uygOL7yS4tccwDnAHwNPUEyUfrut7j8CnyiHPk6z/QTwNxTj98spfrUOdRRLp/ZH2k8pJrR/A3waOML2I+W6oykmWH8FfAc4a4jzO64onx+RdGs5PHUqxZfkY8AxFOPpdZ0G3AHMpxg2OQ/YpExeMyiOmlpB0aP4O4r/x5sAHy1jfpRifuhDg+z/Mor5gqUUSfkbLevGArMpPpdfUyTS04cR+4BZFH/vXwA/oPhbrqxT0faiIeZPLpD0FEVi/hxFb216mVxjBOVEuRh1JL2P4qS+fXsdy2gh6UPAUbab6hFGA9KDiIgRJ2knSfuUQ4WvoTih7Tu9jiuGJ2eBRkQTNqM4Cm0K8DjFuSUX9TKgGL4MMUVERKUMMUVERKUkiIiIqLTRzEGMGzfOkydP7nUYEREblFtuueU3tttPYgU2ogQxefJkFixY0OswIiI2KJJ+Odi6DDFFRESlJIiIiKiUBBEREZWSICIiolISREREVGo0QUiaXt7TdrGkmRXrT5J0h6SFKu67u3vLutPLevdIOrjJOCMi4sUaSxDlDWIuBA4BdgeObk0Apa/Zfr3tPYHzKW5jSbndURT3Ep4OXKRBbl4fERHNaLIHsRew2PYS289RXKxrRusGtp9sWXwpv78z1wzg8vL+x78AFpf7i4iILmnyRLkJrH0f2mXA3u0bSTqZ4kYnmwEHtNT9SVvdCW1VkXQicCLApEmThgxo8sxr6kXewdLZh673PiIiNgQ9n6S2faHt3YCPUdyWcjh1L7Y9zfa08eMrzxSPiIh11GSCWM7aNyqfyNo3fm93OfDudawbEREjrMkEMR+YKmmKpM0oJp3Xui+vpKkti4cC95av5wJHSRoraQrFvYNvbjDWiIho09gchO1Vkk4BrgXGAHNsL5I0C1hgey5wiqQDgecpbu5+XFl3kaRvUty8fhVwsu3VTcUaEREv1ujVXG3PA+a1lZ3Z8vrDHep+Gvh0c9FFREQnPZ+kjoiI/pQEERERlZIgIiKiUhJERERUSoKIiIhKSRAREVEpCSIiIiolQURERKUkiIiIqJQEERERlZIgIiKiUhJERERUSoKIiIhKSRAREVEpCSIiIiolQURERKUkiIiIqJQEERERlZIgIiKiUhJERERUSoKIiIhKSRAREVEpCSIiIiolQURERKUkiIiIqNRogpA0XdI9khZLmlmx/qOS7pJ0u6T/lLRry7rVkhaWj7lNxhkRES+2aVM7ljQGuBA4CFgGzJc01/ZdLZvdBkyz/YykDwHnA0eW6561vWdT8UVERGdN9iD2AhbbXmL7OeByYEbrBrZ/aPuZcvEnwMQG44mIiGEYMkFIukXSyZK2H+a+JwAPtCwvK8sGcwLwvZblzSUtkPQTSe8eZtsREbGe6vQgjgR2phgiulzSwZI0kkFIOhaYBvxTS/GutqcBxwCfk7RbRb0TyySyYMWKFSMZUkTEqDdkgrC92PYZwKuBrwFzgF9KOkfSDh2qLgd2aVmeWJatRdKBwBnA4bZXtrS7vHxeAtwAvKEitottT7M9bfz48UO9lYiIGIZacxCS/hD4Z4pf+N8C3gM8CVzfodp8YKqkKZI2A44C1joaSdIbgM9TJIeHW8q3lzS2fD0O2AdondyOiIiGDXkUk6RbgMeBS4CZLb/yfyppn8Hq2V4l6RTgWmAMMMf2IkmzgAW251IknK2AK8pRq/ttHw68Fvi8pDUUSWx229FPERHRsDqHub6nHOZ5gaQptn9h+887VbQ9D5jXVnZmy+sDB6l3E/D6GrFFRERD6gwxXVmzLCIiNiKD9iAk/QGwB7CtpNaewjbA5k0HFhERvdVpiOk1wLuA7YDDWsqfAj7YYEwREdEHBk0Qtr8LfFfSW2z/dxdjioiIPtBpiOnvbZ8PHCPp6Pb1tk9tNLKIiOipTkNMd5fPC7oRSERE9JdOQ0xXl1dkfb3t07oYU0RE9IGOh7naXk1xFnNERIwydU6UG7hhzxXA0wOFtr/dWFQREdFzdRLE5sAjwAEtZQaSICIiNmJDJgjbx3cjkIiI6C91Ltb3JYoew1psv7+RiCIioi/UGWL695bXmwN/BvyqmXAiIqJf1Bli+lbrsqSvAz9qLKKIiOgLtW4Y1GYq8PKRDiQiIvpLnTmIpyjmIFQ+/xr4WMNxRUREj9UZYtq6G4GMFpNnXrPe+1g6+9ARiCQiorM6k9SU94PYl6IH8V+2r2oyqIiI6L0h5yAkXQScBNwB3AmcJOnCpgOLiIjeqtODOAB4rW0DSPoysKjRqCIioufqHMW0GJjUsrxLWRYRERuxOj2IrYG7Jd1cLr8JWFBewA/bhzcVXERE9E6dBHFm41FERETfqXOY640AkrZp3d72ow3GFRERPVbnRLkTgVnA74A1/P6EuVc2G1pERPRSnSGmvwNeZ/s3TQcTERH9o85RTPcBz6zLziVNl3SPpMWSZlas/6ikuyTdLuk/Je3asu44SfeWj+PWpf2IiFh3dXoQpwM3SfopsHKg0PapnSpJGgNcCBwELAPmS5pr+66WzW4Dptl+RtKHgPOBIyXtAJwFTKMYzrqlrPvYMN5bRESshzoJ4vPA9RRnUq8Zxr73AhbbXgIg6XJgBvBCgrD9w5btfwIcW74+GLhuYCJc0nXAdODrw2g/IiLWQ50E8RLbH12HfU8AHmhZXgbs3WH7E4Dvdag7ob1COYF+IsCkSZPaV0cHuWhgRAylzhzE9ySdKGknSTsMPEYyCEnHUgwn/dNw6tm+2PY029PGjx8/kiFFRIx6dXoQR5fPp7eU1TnMdTnFZTkGTCzL1iLpQOAMYD/bK1vq7t9W94YasUZExAipc6LclHXc93xgqqQpFF/4RwHHtG4g6Q0UcxzTbT/csupa4FxJ25fL72DtBBUREQ0bNEFIOsD29eW9IF7E9rc77dj2KkmnUHzZjwHm2F4kaRawwPZciiGlrYArJAHcb/tw249K+iRFkgGYlTO3IyK6q1MPYj+Ko5cOq1hnoGOCALA9D5jXVnZmy+sDO9SdA8wZqo2IiGjGoAnC9lnl8/HdCyciIvpFnaOYIiJiFEqCiIiISkkQERFRqc55EEh6KzCZte8HcWlDMUVERB+ocz+Iy4DdgIXA6rLYQBJERMRGrE4PYhqwu203HUxERPSPOnMQdwKvaDqQiIjoL3V6EOOAuyTdzNr3gzi8sagiIqLn6iSIs5sOIiIi+k+di/Xd2I1AIiKiv3S6WN+PbO8r6SmKo5ZeWAXY9jaNRxcRET3T6VpM+5bPW3cvnIiI6Bc5kzoiIiolQURERKUkiIiIqFQrQUjatbx3NJK2kJR5iYiIjdyQCULSB4ErKe4dDTARuKrBmCIiog/U6UGcDOwDPAlg+17g5U0GFRERvVfnTOqVtp+TBICkTVn7vIiIdTJ55jXrvY+lsw8dgUgiokqdHsSNkj4ObCHpIOAK4Opmw4qIiF6rkyBmAiuAO4C/BuYBn2gyqIiI6L0612JaA3wB+IKkHYCJuTdERMTGr85RTDdI2qZMDrdQJIrPNh9aRET0Up0hpm1tPwn8OXCp7b2BtzcbVkRE9FqdBLGppJ2AvwT+fTg7lzRd0j2SFkuaWbH+bZJulbRK0hFt61ZLWlg+5g6n3YiIWH91DnOdBVwL/Mj2fEmvBO4dqpKkMcCFwEHAMmC+pLm272rZ7H7gfcBpFbt41vaeNeKLiIgG1JmkvoLi0NaB5SXAX9TY917A4nJ7JF0OzABeSBC2l5br1gwr6oiIaNyQCULS5sAJwB7A5gPltt8/RNUJwAMty8uAvYcR2+aSFgCrgNm2rxpG3YiIWE915iAuA14BHAzcSHEtpqeaDKq0q+1pwDHA5yTt1r6BpBMlLZC0YMWKFV0IKSJi9KiTIF5l+x+Ap21/GTiUej2B5cAuLcsTy7JabC8vn5cANwBvqNjmYtvTbE8bP3583V1HREQNdRLE8+Xz45JeB2xLvYv1zQemSpoiaTPgKKDW0UiStpc0tnw9juJigXd1rhURESOpToK4WNL2wD9QfMHfBZw/VCXbq4BTKI6Auhv4pu1FkmZJOhxA0pskLQPeA3xe0qKy+muBBZJ+BvyQYg4iCSIioovqHMX0xfLljcArh7Nz2/Mort3UWnZmy+v5FENP7fVuAl4/nLYiImJk1bnUxo6SLpH0vXJ5d0knNB9aRET0Up0hpn+jGCbauVz+H+AjDcUTERF9ok6CGGf7m8AaeGFuYXWjUUVERM/VSRBPS3oZ5V3kJL0ZeKLRqCIioufqXIvpoxRHL+0m6cfAeOCIzlUiImJD1zFBlBfc2698vAYQcI/t5zvVi4iIDV/HISbbq4Gjba+yvcj2nUkOERGjQ50hph9LugD4BvD0QKHtWxuLKiIieq5OgtizfJ7VUmbggBGPJiIi+kadM6n/tBuBREREf6lzJvW5krZrWd5e0qcajSoiInquznkQh9h+fGDB9mPAOxuLKCIi+kKdBDFm4NLbAJK2AMZ22D4iIjYCdSapvwr8p6QvlcvHA19uLqSIiOgHdSapzyvvy3BgWfRJ29c2G1ZERPRanR4EFDf8WWX7B5K2lLS17W7clzoiInqkzlFMHwSuBD5fFk0ArmowpoiI6AN1JqlPprgn9JMAtu+l3j2pIyJiA1YnQay0/dzAgqRNKS/9HRERG686CeJGSR8HtpB0EHAFcHWzYUVERK/VSRAzgRXAHcBfA/OATzQZVERE9F6dw1zXAF8oHxERMUoMmiAk3UGHuQbbf9hIRBER0Rc69SDeVT6fXD5fVj4fSyapIyI2eoMmCNu/BJB0kO03tKz6mKRbKeYmIiJiI1VnklqS9mlZeGvNehERsQGr80V/AnCRpKWSlgIXAe+vs3NJ0yXdI2mxpBf1OCS9TdKtklZJOqJt3XGS7i0fx9VpLyIiRk6do5huAf5I0rbl8hN1dixpDHAhcBCwDJgvaa7tu1o2ux94H3BaW90dgLOAaRTzHbeUdR+r03ZERKy/2kNFtp+omxxKewGLbS8pz8S+HJjRts+ltm8H1rTVPRi4zvajZVK4Dpg+jLYjImI9NTmXMAF4oGV5WVk2YnUlnShpgaQFK1asWOdAIyLixTboyWbbF9ueZnva+PHjex1ORMRGpdb9IMojlya3bm/70iGqLQd2aVmeWJbVsRzYv63uDTXrRkTECBgyQUi6DNgNWAisLosNDJUg5gNTJU2h+MI/CjimZlzXAudK2r5cfgdwes26ERExAur0IKYBu9se1tnTtldJOoXiy34MMMf2IkmzgAW250p6E/AdYHvgMEnn2N7D9qOSPkmRZABm2X50OO1HRMT6qZMg7gReATw43J3bnkdx9dfWsjNbXs+nGD6qqjsHmDPcNiMiYmTUSRDjgLsk3QysHCi0fXhjUUVERM/VSRBnNx1ERET0nzpnUt/YjUAiIqK/DHkehKQ3S5ov6beSnpO0WtKT3QguIiJ6p86JchcARwP3AlsAH6C4xlJERGzEap1JbXsxMMb2attfItdFiojY6NWZpH5G0mbAQknnUxzuukFfoiMiIoZW54v+r8rtTgGeprh8xl80GVRERPRenaOYfilpC2An2+d0IaaIiOgDdY5iOoziOkz/US7vKWluw3FFRESP1RliOpvi5j+PA9heCExpLKKIiOgLdRLE8xV3khvWhfsiImLDU+copkWSjgHGSJoKnArc1GxYERHRa3V6EP8b2IPiQn1fB54EPtJgTBER0QfqHMX0DHBG+YiIiFFi0AQx1JFKudx3RMTGrVMP4i3AAxTDSj8F1JWIIiKiL3RKEK8ADqK4UN8xwDXA120v6kZgERHRW4NOUpcX5vsP28cBbwYWAzeU95mOiIiNXMdJakljgUMpehGTgX8FvtN8WBER0WudJqkvBV4HzAPOsX1n16KKiIie69SDOJbi6q0fBk6VXpijFmDb2zQcW0RE9NCgCcJ27vkQETGKJQlERESlJIiIiKjUaIKQNF3SPZIWS5pZsX6spG+U638qaXJZPlnSs5IWlo//22ScERHxYnWu5rpOJI0BLqQ42W4ZMF/SXNt3tWx2AvCY7VdJOgo4DziyXHef7T2bii8iIjprLEFQ3GRose0lAJIuB2YArQliBsUNiQCuBC5Qy+FSEd0weeY161V/6exDRyiSiP7S5BDTBIprOQ1YVpZVbmN7FfAE8LJy3RRJt0m6UdKfNBhnRERUaLIHsT4eBCbZfkTSG4GrJO1h+8nWjSSdCJwIMGnSpB6EGRGx8WqyB7Ec2KVleWJZVrmNpE2BbYFHbK+0/QiA7VuA+4BXtzdg+2Lb02xPGz9+fANvISJi9GoyQcwHpkqaImkz4Cig/R4Tc4HjytdHANfbtqTx5SQ3kl4JTAWWNBhrRES0aWyIyfaq8sqv1wJjgDm2F0maBSywPRe4BLhM0mLgUYokAvA2YJak54E1wEm2H20q1oheW9+JcshkeYy8RucgbM+juNhfa9mZLa9/B7ynot63gG81GVtERHSWM6kjIqJSEkRERFRKgoiIiEpJEBERUSkJIiIiKiVBREREpSSIiIiolAQRERGVkiAiIqJSEkRERFRKgoiIiEpJEBERUSkJIiIiKiVBREREpSSIiIiolAQRERGVkiAiIqJSEkRERFRKgoiIiEpJEBERUSkJIiIiKm3a6wAion9MnnnNetVfOvvQEYok+kF6EBERUSkJIiIiKiVBREREpSSIiIio1OgktaTpwL8AY4Av2p7dtn4scCnwRuAR4EjbS8t1pwMnAKuBU21f22SsEdEf1neiHDJZPlIa60FIGgNcCBwC7A4cLWn3ts1OAB6z/Srgs8B5Zd3dgaOAPYDpwEXl/iIiokuaHGLaC1hse4nt54DLgRlt28wAvly+vhJ4uySV5ZfbXmn7F8Dicn8REdElTQ4xTQAeaFleBuw92Da2V0l6AnhZWf6TtroT2huQdCJwYrn4W0n3rGfM44DfdNpA561nCyMQRz/E0C9xjKIY+iWOfoihX+IY8v9Il6xvHLsOtmKDPlHO9sXAxSO1P0kLbE8bqf1tyHH0Qwz9Ekc/xNAvcfRDDP0SRz/E0HQcTQ4xLQd2aVmeWJZVbiNpU2BbisnqOnUjIqJBTSaI+cBUSVMkbUYx6Ty3bZu5wHHl6yOA6227LD9K0lhJU4CpwM0NxhoREW0aG2Iq5xROAa6lOMx1ju1FkmYBC2zPBS4BLpO0GHiUIolQbvdN4C5gFXCy7dVNxdpixIar1lM/xNEPMUB/xNEPMUB/xNEPMUB/xNEPMUCDcaj4wR4REbG2nEkdERGVkiAiIqJSEkRERFQatQlC0maS3ivpwHL5GEkXSDpZ0ku6GMcrJZ0m6V8kfUbSSZK26Vb7HeK6tAdt7iXpTeXr3SV9VNI7ux1Hr0nae+DfgKQtJJ0j6WpJ50nattfx9ZKkfct/F+/ocrunStpl6C0bj+MPJL1d0lZt5dMbaW+0TlJL+irFUVxbAo8DWwHfBt5O8bkcN3jtEYvhVOBdwP8D3gncVsbyZ8Df2L6h6RjKONoPPxbwp8D1ALYP70IMZ1Fct2tT4DqKs+5/CBwEXGv7003HMBRJx9v+UhfaWQT8UXkk4MXAM5SXoinL/7zpGPqFpJtt71W+/iBwMvAd4B3A1e0XAG0wjieAp4H7gK8DV9he0Y22W2I4leL93w3sCXzY9nfLdbfa/uMRb9T2qHwAt5fPmwIPAWPKZQ2s60IMd7S0uyVwQ/l6EnBbFz+LW4GvAPsD+5XPD5av9+vmZ1F+Dk8C25TlW3Tr71Ejxvu71M7drX+btnULu/h+twVmAz+nOAz9EYovp9nAdl2K4baW1/OB8eXrlwJ3dPGzuI1ixOUdFIfnrwD+g+I8rq27FMMdwFbl68nAAooksdbnNJKPUTvEBGxSnsC3NcWX0kDXfSzQtSEmfn8uyliKXgy27+9yDNOAW4AzgCdc9FyetX2j7Ru7FMMq26ttPwPcZ/tJANvPAmu6FAOSbh/kcQewY5fCuFPS8eXrn0maVsb2auD5LsUA8E3gMWB/2zvYfhlFz/Kxcl03bCJpe0kvo+jZrwCw/TTFOVLdYttrbH/f9gnAzsBFFFebXtKlGDax/dsymKUUP+QOkfQZih+2I26DvhbTerqE4pfRGIovxiskLQHeTHHl2W74IjBf0k+BP+H3lzsfT/GLrStsrwE+K+mK8vkhuv9v4zlJW5YJ4o0DheWYe9cSBEUSOJjiS7CVgJu6FMMHgH+R9AmKi7D9t6QHKC5s+YEuxQAw2fZal72z/WvgPEnv71IM21L8eBFgSTvZfrAcg2/kS3EQa7Vl+3mKKz7MlbRll2J4SNKetheWMfxW0ruAOcDrm2hw1M5BAEjaGcD2ryRtBxxIMYzQtct6SNoDeC1wp+2fd6vdTiQdCuxj++NdbHOs7ZUV5eOAnWzf0aU4LgG+ZPtHFeu+ZvuYbsRRtrcNMIUiWS+z/VC32i7b/z7wA+DLA21L2hF4H3CQ7QO7GU9bbFsCO7q4HUA32nu17f/pRlsdYphI0dP+dcW6fWz/eMTbHM0JIiIGJ2l7YCbF/VleXhY/RPHLebbt9l5WbGSSICJi2Lp1RFf0VhJERAybpPttT+p1HNGs0TxJHREdSLp9sFV074iu6KEkiIgYTD8c0RU9lAQREYP5d4oTsxa2r5B0Q9ejia7LHERERFQazWdSR0REB0kQERFRKQkiIiIqJUFERESlJIiIiKj0/wE7qDbSvKC15AAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "rf_importances = rfc3.feature_importances_\n",
        "# rf_std = np.std([tree.feature_importances_ for tree in model_rf.best_estimator_.estimators_], axis=0)\n",
        "\n",
        "feature_names = [f\"{i}\" for i in list([1,2,3,4,5,6,7,8,9,10])]\n",
        "forest_importances = pd.Series(rf_importances, index=feature_names)\n",
        "most_important = forest_importances[forest_importances > 0.00]\n",
        "most_important = most_important.sort_values(ascending=False)\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "most_important.plot.bar(ax=ax)\n",
        "ax.set_title(\"Feature importances using MDI\")\n",
        "ax.set_ylabel(\"Mean decrease in impurity\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iXgQTArK3uoK"
      },
      "source": [
        "## Support Vector Machine\n",
        "SVC(C= 10.0,degree= 2, kernel= 'rbf', probability = True) are determined to be the best parameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 138,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ajNvM7e8MiwM",
        "outputId": "39192dc0-77d9-4f47-98c8-6503433d2f83"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 18 candidates, totalling 54 fits\n",
            "[CV 1/3] END ....C=0.1, degree=2, kernel=linear;, score=0.992 total time=   0.0s\n",
            "[CV 2/3] END ....C=0.1, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ....C=0.1, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=0.1, degree=2, kernel=poly;, score=0.996 total time=   0.0s\n",
            "[CV 2/3] END ......C=0.1, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=0.1, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .......C=0.1, degree=2, kernel=rbf;, score=0.998 total time=   0.2s\n",
            "[CV 2/3] END .......C=0.1, degree=2, kernel=rbf;, score=1.000 total time=   0.2s\n",
            "[CV 3/3] END .......C=0.1, degree=2, kernel=rbf;, score=1.000 total time=   0.2s\n",
            "[CV 1/3] END ....C=0.1, degree=3, kernel=linear;, score=0.992 total time=   0.0s\n",
            "[CV 2/3] END ....C=0.1, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ....C=0.1, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=0.1, degree=3, kernel=poly;, score=0.992 total time=   0.0s\n",
            "[CV 2/3] END ......C=0.1, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=0.1, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .......C=0.1, degree=3, kernel=rbf;, score=0.998 total time=   0.2s\n",
            "[CV 2/3] END .......C=0.1, degree=3, kernel=rbf;, score=1.000 total time=   0.2s\n",
            "[CV 3/3] END .......C=0.1, degree=3, kernel=rbf;, score=1.000 total time=   0.2s\n",
            "[CV 1/3] END ....C=1.0, degree=2, kernel=linear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END ....C=1.0, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ....C=1.0, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=1.0, degree=2, kernel=poly;, score=0.996 total time=   0.0s\n",
            "[CV 2/3] END ......C=1.0, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=1.0, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .......C=1.0, degree=2, kernel=rbf;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END .......C=1.0, degree=2, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END .......C=1.0, degree=2, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ....C=1.0, degree=3, kernel=linear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END ....C=1.0, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ....C=1.0, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=1.0, degree=3, kernel=poly;, score=0.996 total time=   0.0s\n",
            "[CV 2/3] END ......C=1.0, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=1.0, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .......C=1.0, degree=3, kernel=rbf;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END .......C=1.0, degree=3, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END .......C=1.0, degree=3, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ...C=10.0, degree=2, kernel=linear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END ...C=10.0, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ...C=10.0, degree=2, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .....C=10.0, degree=2, kernel=poly;, score=0.993 total time=   0.0s\n",
            "[CV 2/3] END .....C=10.0, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END .....C=10.0, degree=2, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=10.0, degree=2, kernel=rbf;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END ......C=10.0, degree=2, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=10.0, degree=2, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ...C=10.0, degree=3, kernel=linear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END ...C=10.0, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ...C=10.0, degree=3, kernel=linear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END .....C=10.0, degree=3, kernel=poly;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END .....C=10.0, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END .....C=10.0, degree=3, kernel=poly;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END ......C=10.0, degree=3, kernel=rbf;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END ......C=10.0, degree=3, kernel=rbf;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END ......C=10.0, degree=3, kernel=rbf;, score=1.000 total time=   0.0s\n"
          ]
        }
      ],
      "source": [
        "from sklearn.svm import SVC\n",
        "\n",
        "svm_parameters = {'C': np.logspace(-1, 1, 3),\n",
        "                  'kernel': ('linear', 'poly','rbf'),\n",
        "                  'degree': [2,3]}\n",
        "\n",
        "svm = GridSearchCV(estimator=SVC(probability=True),\n",
        "                         param_grid= svm_parameters,\n",
        "                         cv = 3,\n",
        "                         scoring='roc_auc',\n",
        "                         verbose = 3).fit(x_train_bowandtfidf,y_train)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 139,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "aJLrz5ZWa3Zl",
        "outputId": "ee43cddd-02aa-405f-9d3c-5330d26ff6f5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9993785732040765\n",
            "Precision Score: 0.9900124843945068\n",
            "Recall Score: 0.9974842767295597\n",
            "F1 Score: 0.9937343358395989\n",
            "ROC Score: 0.9992726407076983\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A134463490>\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAoVklEQVR4nO3de5xd873/8ddbJBKSCAkOucgQigSpjCglLqlK0eDnkqTV4mgdBKdUTjntj9DqzeUUVUp5uDQSt6poNX4aEnUJSYg0iapIowatSJ2QJhFJP78/1pqxZ7Jn9prL2mNn3s/HYx5Zl+9a67P23tmfvb7ftb5fRQRmZtZxbdLeAZiZWftyIjAz6+CcCMzMOjgnAjOzDs6JwMysg9u0vQNorj59+sTAgQPbOwwzs4oyd+7cdyNim2LrKi4RDBw4kDlz5rR3GGZmFUXS642tc9WQmVkH50RgZtbBORGYmXVwTgRmZh2cE4GZWQeXWyKQdJukdyQtaGS9JF0nabGk+ZL2ySsWMzNrXJ5XBLcDo5pY/wVgl/TvDODGHGMxM7NG5PYcQUQ8KWlgE0WOAe6MpB/sWZJ6Sdo+It7OK6a/v7+Gxe+sbBDnhuWCyFCmyLIGBYt28N3S4zVYluX4xcoV73W8yHaZjtewTOn9FI+pdFfoWd6nojHl+N61/HyLlWnZZ65hwSzvU7Ks+dtl/cxlOn6ZP3Mblmmb4xeLoaXvXRYjd9+Ovfv3avZ2pbTnA2V9gTcK5mvSZRskAklnkFw1MGDAgBYdbNaS5Yy9eVaLtjUzy4PUvPLb9uy60SWCzCLiZuBmgOrq6haNpPPXf6wC4LLRg9l9+5711hV7MxouKv6GbbiwYblim6nIzrIcTw1KZf0QbRhT6bhbul3x8y0aVenjb7Cf0q9b8ZiynW+WMg1jyHq+Wd67DRZl2E+xfeX5mSsaU6W8d+X8zDX3G76dtWcieBPoXzDfL12Wq5G7b0u/rTbP+zBmZhWjPW8fnQp8Nb176DPAijzbB8zMrLjcrggkTQYOAfpIqgEuBToDRMRNwCPAkcBiYBVwWl6xmJlZ4/K8a2hcifUBjM/r+GZmlk3HebK4RU3MZmYbv46TCFKV1ppvZpa3DpcIzMysPicCM7MOzonAzKyDcyIwM+vgMt0+KmkTYG9gB2A1sCAi3skzMDMzK48mE4GknYFvAZ8DXgWWAV2BXSWtAn4O3BER/8o7UDMzy0epK4LvkYwT8B/RoN9WSdsCXwK+AtyRT3hmZpa3JhNBU08Hp1VDP2nrgMzMrLxa3Fgs6fC2DMTMzNpHa+4aurXNoiiDYiMZmZlZ6cbiqY2tAnq3fTj5cwcTZmb1lWosPgg4GVjZYLmA4blEZGZmZVUqEcwCVkXEzIYrJL2ST0hmZlZOpe4a+kIT60a0fThmZlZu7mLCzKyDcyIwM+vgnAjMzDo4JwIzsw4ucyKQNLGpeTMzq0zNuSKYW2LezMwqUOZEEBEPNzVvZmaVqVQXE9dD4530RMR5bR5RTsJdDZmZFVXqyeI5ZYmijOTOhszM6in1ZHG9AWckbR4Rq/INyczMyilTG4Gk/SUtAv6Uzu8t6We5RmZmZmWRtbH4J8ARwHKAiHgJcF9DZmYbgebcNfRGg0Xr2zgWMzNrB6Uai2u9IekAICR1Bv4TeDm/sMzMrFyyXhGcCYwH+gJvAUPTeTMzq3CZEkFEvBsRX46I7SJim4g4OSKWl9pO0ihJr0haLOmiIusHSHpC0ouS5ks6siUnYWZmLZf1rqGdJD0saZmkdyQ9JGmnEtt0Am4AvgDsAYyTtEeDYt8B7o2ITwNjAd+JZGZWZlmrhu4G7gW2B3YA7gMml9hmOLA4IpZExFpgCnBMgzIB9EyntySpdjIzszLKmgg2j4i7ImJd+vdLoGuJbfoChXca1aTLCk0ETpZUAzwCnFtsR5LOkDRH0pxly5ZlDLk+9zBhZlZck4lA0taStgZ+J+kiSQMl7Sjpv0i+uFtrHHB7RPQDjgTukrRBTBFxc0RUR0T1Ntts06oDCvcxYWZWqNTto3NJfkzXfnv+R8G6AC5uYts3gf4F8/3SZYVOB0YBRMSzkroCfYB3SsRlZmZtpFRfQ1Wt2PdsYBdJVSQJYCzwpQZl/gqMBG6XtDtJdVPL6n7MzKxFsj5QhqQhJHf/1LUNRMSdjZWPiHWSzgEeBToBt0XEQkmXA3MiYirwTeAWSeeTXGGcGuEOo83MyilTIpB0KXAISSJ4hOSW0KeARhMBQEQ8QoO2hIi4pGB6EfDZZkVsZmZtKutdQyeQVOH8LSJOA/Ymud3TzMwqXNZEsDoi/gWsk9STpDG3f4ltzMysAmRtI5gjqRdwC8mdRCuBZ/MKyszMyidTIoiIs9PJmyRNA3pGxPz8wjIzs3IpNXj9Pk2ti4gX2j4kMzMrp1JXBFc3sS6Aw9owllz5plQzs+JKPVB2aLkCKRe5hwkzs3oyD1VpZmYbJycCM7MOzonAzKyDyzpCmSSdLOmSdH6ApOH5hmZmZuWQ9YrgZ8D+JOMHAHxAMgylmZlVuKxPFu8XEftIehEgIt6T1CXHuMzMrEyyXhF8lA5GHwCStgH+lVtUZmZWNlkTwXXAg8C2kq4g6YL6+7lFZWZmZZO1r6FJkuaSdEUt4NiIeDnXyMzMrCyyDkxzHTAlItxAbGa2kclaNTQX+I6k1yRdJak6z6DMzKx8MiWCiLgjIo4E9gVeAX4k6dVcIzMzs7Jo7pPFg4DdgB2BP7V9OGZmVm5Znyz+cXoFcDmwAKiOiC/mGpmZmZVF1gfKXgP2j4h38wzGzMzKr9QIZbtFxJ+A2cAASQMK13uEMjOzylfqiuAC4AyKj1RWUSOUmZlZcaVGKDsjnfxCRKwpXCepa25R5SDwWJVmZsVkvWvomYzLPvE8UqWZWX2l2gj+DegLdJP0aT7+Hu0JbJ5zbGZmVgal2giOAE4F+gHXFCz/APjvnGIyM7MyKtVGcAdwh6TjI+KBMsWUi3ATgZlZUaWqhk6OiF8CAyVd0HB9RFxTZLNPNjcSmJnVU6qxeIv03+5AjyJ/TZI0StIrkhZLuqiRMidJWiRpoaS7mxG7mZm1gVJVQz9P/72suTtORzS7ATgcqAFmS5oaEYsKyuwCXAx8Nh3+ctvmHsfMzFqnOX0N9ZTUWdJ0ScsknVxis+HA4ohYEhFrgSnAMQ3KfB24ISLeA4iId5p7AmZm1jpZnyP4fES8DxwNLCXphXRCiW36Am8UzNekywrtCuwq6WlJsySNKrYjSWdImiNpzrJlyzKGXJ/bis3MisuaCGqrkI4C7ouIFW10/E2BXYBDgHHALZJ6NSwUETdHRHVEVG+zzTatOqDcWmxmVk/WRPAbSX8ChgHTJW0DrCmxzZtA/4L5fumyQjXA1Ij4KCL+AvyZJDGYmVmZZB2h7CLgAJJxCD4C/smG9f0NzQZ2kVQlqQswFpjaoMyvSa4GkNSHpKpoSdbgzcys9bIOXt8ZOBkYIQlgJnBTU9tExDpJ5wCPAp2A2yJioaTLgTkRMTVd93lJi4D1wISIWN7is2k6oFx2a2ZW6bIOTHMj0Bn4WTr/lXTZ15raKCIeAR5psOySgukg6ep6g4fV8iI3EZiZ1ZM1EewbEXsXzD8u6aU8AjIzs/LK2li8XtLOtTOSdiKpyjEzswqX9YpgAvCEpCUkvfXsCJyWW1Q5cAuBmVlxJRNBeqvoCpInhWu7gHglIj7MM7C8uInAzKy+JquGJH0NWAhcD8wDBkbE/EpNAmZmtqFSVwTfAAZHxLK0XWASGz4LYGZmFaxUY/HaiFgGEBFLgM3yDykffozAzKy4UlcE/SRd19h8RJyXT1j5kR8kMDOrp1QiaNjD6Ny8AjEzs/aRZcxiMzPbiJW6a+gWSUMaWbeFpH+X9OV8Qmtb4UYCM7OiSlUN3QBcImlPYAGwDOhK0lV0T+A2kjuJKoZbCMzM6itVNTQPOElSd6Aa2B5YDbwcEa/kH56ZmeUtUxcTEbESmJFvKGZm1h6ydjpX8dxCYGZWXIdJBLX8GIGZWX3NSgSSNs8rEDMzax+ZEoGkA9LhJP+Uzu8t6WclNjMzswqQ9Yrgf4AjgOUAEfESMCKvoPLgxwjMzIrLXDUUEW80WFSRI5TJTxKYmdWTdYSyNyQdAISkzsB/Ai/nF5aZmZVL1iuCM4HxQF/gTWAocHZOMZmZWRllvSL4VETU61NI0meBp9s+pHy4icDMrLisVwTXZ1z2yecmAjOzepq8IpC0P3AAsI2kCwpW9QQ65RmYmZmVR6mqoS5A97Rcj4Ll7wMn5BWUmZmVT6neR2cCMyXdHhGvlymmXHg8AjOz4rI2Fq+SdCUwmGQ8AgAi4rBcosqR+xoyM6sva2PxJJLuJaqAy4ClwOycYjIzszLKmgh6R8StwEcRMTMi/h2ouKsBMzPbUNaqoY/Sf9+WdBTwFrB1PiGZmVk5Zb0i+J6kLYFvAhcCvwC+UWojSaMkvSJpsaSLmih3vKSQVJ0xnhZzE4GZWX1Zh6r8TTq5AjgU6p4sbpSkTsANwOFADTBb0tSIWNSgXA+Svouea17oZmbWFpq8IpDUSdI4SRdKGpIuO1rSM8BPS+x7OLA4IpZExFpgCnBMkXLfBX4ErGl++GZm1lqlqoZuBb4G9Aauk/RL4CrgxxHx6RLb9gUKu66uSZfVkbQP0D8iftvUjiSdIWmOpDnLli0rcdji/BiBmVlxpaqGqoG9IuJfkroCfwN2jojlrT2wpE2Aa4BTS5WNiJuBmwGqq6tb9ZUuP0hgZlZPqSuCtRHxL4CIWAMsaUYSeBPoXzDfL11WqwcwBJghaSnwGWBqORqMzczsY6WuCHaTND+dFrBzOi8gImKvJradDewiqYokAYwFvlS7MiJWAH1q5yXNAC6MiDnNPgszM2uxUolg95buOCLWSToHeJSkp9LbImKhpMuBORExtaX7blE8HpHAzKyoUp3OtaqjuYh4BHikwbJLGil7SGuOlZVbCMzM6ss8eL2ZmW2cnAjMzDq4zIlAUjdJn8ozmDz5OQIzs+IyJQJJXwTmAdPS+aGSytrY21b8GIGZWX1ZrwgmknQZ8b8AETGPZGwCMzOrcFkTwUfpff+FXNliZrYRyDoewUJJXwI6SdoFOA94Jr+wzMysXLJeEZxLMl7xh8DdJN1RfyOnmMzMrIyyXhHsFhHfBr6dZzBmZlZ+Wa8Irpb0sqTv1o5LYGZmG4dMiSAiDiUZmWwZ8HNJf5T0nVwjMzOzssj8QFlE/C0irgPOJHmmoGifQWZmVlmyPlC2u6SJkv4IXE9yx1C/XCMzM7OyyNpYfBtwD3BERLyVYzxmZlZmmRJBROyfdyBmZtY+mkwEku6NiJPSKqHCJ4mzjFBmZmYVoNQVwX+m/x6ddyBmZtY+mmwsjoi308mzI+L1wj/g7PzDMzOzvGW9ffTwIsu+0JaBmJlZ+yjVRnAWyS//nSTNL1jVA3g6z8DMzKw8SrUR3A38DvgBcFHB8g8i4h+5RWVmZmVTKhFERCyVNL7hCklbOxmYmVW+LFcERwNzSW4fLRzoMYCdcorLzMzKpMlEEBFHp/96WEozs41U1r6GPitpi3T6ZEnXSBqQb2hmZlYOWW8fvRFYJWlv4JvAa8BduUVlZmZlkzURrIuIAI4BfhoRN5DcQmpmZhUua++jH0i6GPgKcJCkTYDO+YVlZmblkvWKYAzJwPX/HhF/IxmL4MrcojIzs7LJOlTl34BJwJaSjgbWRMSduUZmZmZlkfWuoZOA54ETgZOA5ySdkGG7UZJekbRY0kVF1l8gaZGk+ZKmS9qxuSdgZmatk7WN4NvAvhHxDoCkbYDfA/c3toGkTsANJB3W1QCzJU2NiEUFxV4EqiNiVdqv0Y9JqqHMzKxMsrYRbFKbBFLLM2w7HFgcEUsiYi0wheSuozoR8URErEpnZ+FxkM3Myi7rFcE0SY8Ck9P5McAjJbbpC7xRMF8D7NdE+dNJOrjbgKQzgDMABgzwc2xmZm0p65jFEyT9H+DAdNHNEfFgWwUh6WSgGji4kePfDNwMUF1dHcXKmJlZy5Qaj2AX4CpgZ+CPwIUR8WbGfb8J9C+Y75cua3iMz5G0QRwcER9m3LeZmbWRUvX8twG/AY4n6YH0+mbsezawi6QqSV2AscDUwgKSPg38HBjdoA3CzMzKpFTVUI+IuCWdfkXSC1l3HBHrJJ0DPAp0Am6LiIWSLgfmRMRUkofSugP3SQL4a0SMbvZZmJlZi5VKBF3TX+214xB0K5yPiCYTQ0Q8QoNG5Yi4pGD6c82O2MzM2lSpRPA2cE3B/N8K5gM4LI+gzMysfEoNTHNouQIxM7P2kfWBMjMz20g5EZiZdXBOBGZmHVzW3keVjlV8STo/QNLwfEMzM7NyyHpF8DNgf2BcOv8BSc+iZmZW4bJ2OrdfROwj6UWAiHgvfVrYzMwqXNYrgo/S8QUC6sYj+FduUZmZWdlkTQTXAQ8C20q6AngK+H5uUZmZWdlk7YZ6kqS5wEiS7iWOjYiXc43MzMzKIlMikDQAWAU8XLgsIv6aV2BmZlYeWRuLf0vSPiCgK1AFvAIMzikuMzMrk6xVQ3sWzkvaBzg7l4jMzKysWvRkcdr9dFPjD5uZWYXI2kZwQcHsJsA+wFu5RGRmZmWVtY2gR8H0OpI2gwfaPhwzMyu3kokgfZCsR0RcWIZ4zMyszJpsI5C0aUSsBz5bpnjMzKzMSl0RPE/SHjBP0lTgPuCftSsj4lc5xmZmZmWQtY2gK7CcZIzi2ucJAnAiMDOrcKUSwbbpHUML+DgB1IrcojL7hPjoo4+oqalhzZo17R2KWSZdu3alX79+dO7cOfM2pRJBJ6A79RNALScC2+jV1NTQo0cPBg4ciFTsv4HZJ0dEsHz5cmpqaqiqqsq8XalE8HZEXN660Mwq15o1a5wErGJIonfv3ixbtqxZ25V6stiffuvwnASskrTk81oqEYxsWShmZlYpmkwEEfGPcgViZsX9/e9/50tf+hI77bQTw4YNY//99+fBBx8sWvatt97ihBNOKLrukEMOYc6cOQDcdttt7Lnnnuy1114MGTKEhx56KLf4ly5dypAhQxpdf9VVV7HbbrsxdOhQ9t13X+68804uu+wyLr744nrl5s2bx+677150HyeccAJLliypV1YS06ZNazKOiRMnctVVVzUZS2uNGjWKXr16cfTRRzda5sMPP2TMmDEMGjSI/fbbj6VLl9at+8EPfsCgQYP41Kc+xaOPPgrA2rVrGTFiBOvWrWt1fNDCTufMrDwigmOPPZYRI0awZMkS5s6dy5QpU6ipqdmg7Lp169hhhx24//77m9xnTU0NV1xxBU899RTz589n1qxZ7LXXXq2OtSVfSjfddBOPPfYYzz//PPPmzWP69OlEBOPGjeOee+6pV3bKlCmMGzdug30sXLiQ9evXs9NOO9Utmzx5MgceeCCTJ09udSytNWHCBO66664my9x6661stdVWLF68mPPPP59vfetbACxatIgpU6awcOFCpk2bxtlnn8369evp0qULI0eO3OA1aqmszxGYdXiXPbyQRW+936b73GOHnlz6xcaH9Xj88cfp0qULZ555Zt2yHXfckXPPPReA22+/nV/96lesXLmS9evXc8cdd3D00UezYMECVq9ezWmnncZLL73EbrvtxurVqwF455136NGjB927dwege/fuddOvvfYa48ePZ9myZWy++ebccsst7Lbbbjz88MN873vfY+3atfTu3ZtJkyax3XbbMXHiRF577TWWLFnCgAED+MlPfsKZZ55Z9+v8xhtvZIcddmD9+vV8/etf55lnnqFv37489NBDdOvWje9///vMmDGDnj17AtCzZ09OOeUUALbaaiuee+459tsv6ej43nvvrftFXGjSpEkcc8wxdfMRwX333cdjjz3GQQcdxJo1a+jatWvJ96KpWFpj5MiRzJgxo8kyDz30EBMnTgSSq5tzzjmHiOChhx5i7NixbLbZZlRVVTFo0CCef/559t9/f4499lguvvhivvzlL7c6Rl8RmH2CLVy4kH322afJMi+88AL3338/M2fOrLf8xhtvZPPNN+fll1/msssuY+7cuQDsvffebLfddlRVVXHaaafx8MN1Aw9yxhlncP311zN37lyuuuoqzj47GXbkwAMPZNasWbz44ouMHTuWH//4x3XbLFq0iN///vdMnjyZ8847j4MPPpiXXnqJF154gcGDkyT36quvMn78eBYuXEivXr144IEHeP/99/nggw/q/ZIvNG7cOKZMmQLArFmz2Hrrrdlll102KPf0008zbNiwuvlnnnmGqqoqdt55Zw455BB++9vfNvn6ASVjKXTllVcydOjQDf7OO++8kts25s0336R///4AbLrppmy55ZYsX7683nKAfv368eabbwIwZMgQZs+e3eJjFvIVgVlGTf1yL5fx48fz1FNP0aVLl7ovgcMPP5ytt956g7JPPvlk3ZfTXnvtVVf906lTJ6ZNm8bs2bOZPn06559/PnPnzuXCCy/kmWee4cQTT6zbx4cffggk1Uljxozh7bffZu3atfXuUR89ejTdunUDkiuY2nr1Tp06seWWW/Lee+9RVVXF0KFDARg2bFi9OvDGjBkzhgMOOICrr7660WohgLfffpttttmmbn7y5MmMHTsWgLFjx3LnnXdy/PHHN3o3TXPvspkwYQITJkxo1jZ56NSpE126dOGDDz6gR48epTdoQq6JQNIo4FqSB9N+ERE/bLB+M+BOYBhJFxZjImJpnjGZVZLBgwfzwAMf9/h+ww038O6771JdXV23bIsttmj2fiUxfPhwhg8fzuGHH85pp53GBRdcQK9evZg3b94G5c8991wuuOACRo8ezYwZM+qqMbIef7PNNqub7tSpE6tXr6Znz550796dJUuWFP0l3r9/f6qqqpg5cyYPPPAAzz77bNF9d+vWre7J7/Xr1/PAAw/w0EMPccUVV9Q9YPXBBx/Qu3dv3nvvvXrb/uMf/6CqqqpkLIWuvPJKJk2atMHyESNGcN1115V8LYrp27cvb7zxBv369WPdunWsWLGC3r171y2vVVNTQ9++fevmP/zww0zVXqXkVjWUdl99A/AFYA9gnKQ9GhQ7HXgvIgYB/wP8KK94zCrRYYcdxpo1a7jxxhvrlq1atSrTtiNGjODuu+8GYMGCBcyfPx9I7ix64YUX6srNmzePHXfckZ49e1JVVcV9990HJHXtL730EgArVqyo+wK64447Gj3myJEj62Jdv349K1asaDLGiy++mPHjx/P++0nby8qVK+vdqTNu3DjOP/98dtppJ/r161d0H7vvvjuLFy8GYPr06ey111688cYbLF26lNdff53jjz+eBx98kO7du7P99tvz+OOPA0kSmDZtGgceeGCmWGpNmDCBefPmbfDX0iQAyVVV7et6//33c9hhhyGJ0aNHM2XKFD788EP+8pe/8OqrrzJ8+HAAli9fTp8+fZrVlURj8mwjGA4sjoglEbEWmAIc06DMMUDtp+p+YKT89I5ZHUn8+te/ZubMmVRVVTF8+HBOOeUUfvSj0r+ZzjrrLFauXMnuu+/OJZdcUleP/tFHH3HhhRfW3SZ5zz33cO211wJJw+utt97K3nvvzeDBg+tuK504cSInnngiw4YNo0+fPo0e89prr+WJJ55gzz33ZNiwYSxatKhkjIceeij77rsvQ4YM4aCDDmKTTT7+WjrxxBNZuHBho9VCAEcddVRdY+zkyZM57rjj6q0//vjj6+4euvPOO/nud7/L0KFDOeyww7j00kvZeeedM8XSUgcddBAnnngi06dPp1+/fnUN3pdccglTp04F4PTTT2f58uUMGjSIa665hh/+MKk8GTx4MCeddBJ77LEHo0aN4oYbbqBTp04APPHEExx11FGtjg9Isn4ef8AJJNVBtfNfAX7aoMwCoF/B/GtAnyL7OgOYA8wZMGBAtMSjC96Os345J1avXdei7a1jWrRoUXuHYCWsWrUq9ttvv1i3rmP93z7uuOPilVdeKbqu2OcWmBONfF9XxF1DEXFzRFRHRHVho1BzfH7wv/GzLw+ja+dObRydmbWnbt26cdlll9XdTdMRrF27lmOPPZZdd921TfaXZ2Pxm0D/gvl+6bJiZWokbQpsSdJobGaW2RFHHNHeIZRVly5d+OpXv9pm+8vzimA2sIukKkldgLHA1AZlpgK1T2ycADyeXsKYfWL4I2mVpCWf19wSQUSsA84BHgVeBu6NiIWSLpc0Oi12K9Bb0mLgAuCivOIxa4muXbuyfPlyJwOrCJHeLtvcW0pVaR/w6urqqO04yyxvHqHMKk1jI5RJmhsR1cW28ZPFZk3o3Llzs0Z6MqtEFXHXkJmZ5ceJwMysg3MiMDPr4CqusVjSMuD1Fm7eB3i3DcOpBD7njsHn3DG05px3jIiiT+RWXCJoDUlzGms131j5nDsGn3PHkNc5u2rIzKyDcyIwM+vgOloiuLm9A2gHPueOwefcMeRyzh2qjcDMzDbU0a4IzMysAScCM7MObqNMBJJGSXpF0mJJG/RoKmkzSfek65+TNLAdwmxTGc75AkmLJM2XNF3Sju0RZ1sqdc4F5Y6XFJIq/lbDLOcs6aT0vV4o6e5yx9jWMny2B0h6QtKL6ef7yPaIs61Iuk3SO5IWNLJekq5LX4/5kvZp9UEbG7qsUv+ATiRDXu4EdAFeAvZoUOZs4KZ0eixwT3vHXYZzPhTYPJ0+qyOcc1quB/AkMAuobu+4y/A+7wK8CGyVzm/b3nGX4ZxvBs5Kp/cAlrZ33K085xHAPsCCRtYfCfwOEPAZ4LnWHnNjvCIYDiyOiCURsRaYAhzToMwxwB3p9P3ASEkqY4xtreQ5R8QTEbEqnZ1FMmJcJcvyPgN8F/gRsDH0I53lnL8O3BAR7wFExDtljrGtZTnnAHqm01sCb5UxvjYXEU8C/2iiyDHAnZGYBfSStH1rjrkxJoK+wBsF8zXpsqJlIhlAZwXQuyzR5SPLORc6neQXRSUrec7pJXP/iPhtOQPLUZb3eVdgV0lPS5olaVTZostHlnOeCJwsqQZ4BDi3PKG1m+b+fy/J4xF0MJJOBqqBg9s7ljxJ2gS4Bji1nUMpt01JqocOIbnqe1LSnhHxv+0ZVM7GAbdHxNWS9gfukjQkIv7V3oFVio3xiuBNoH/BfL90WdEykjYluZxcXpbo8pHlnJH0OeDbwOiI+LBMseWl1Dn3AIYAMyQtJalLnVrhDcZZ3ucaYGpEfBQRfwH+TJIYKlWWcz4duBcgIp4FupJ0zraxyvT/vTk2xkQwG9hFUpWkLiSNwVMblJkKnJJOnwA8HmkrTIUqec6SPg38nCQJVHq9MZQ454hYERF9ImJgRAwkaRcZHRGVPM5pls/2r0muBpDUh6SqaEkZY2xrWc75r8BIAEm7kySCZWWNsrymAl9N7x76DLAiIt5uzQ43uqqhiFgn6RzgUZI7Dm6LiIWSLgfmRMRU4FaSy8fFJI0yY9sv4tbLeM5XAt2B+9J28b9GxOh2C7qVMp7zRiXjOT8KfF7SImA9MCEiKvZqN+M5fxO4RdL5JA3Hp1byDztJk0mSeZ+03eNSoDNARNxE0g5yJLAYWAWc1upjVvDrZWZmbWBjrBoyM7NmcCIwM+vgnAjMzDo4JwIzsw7OicDMrINzIugAJK2XNK/gb2ATZVe2wfFul/SX9FgvpE97Nncfv5C0Rzr93w3WPdPaGNP91L4uCyQ9LKlXifJDW9KzpaTtJf0mnT5E0or0uC9LurQF+xtd2wunpGNrX6d0/vL0wcFWSd/DE0qUmdGcB/TSc/9NhnJFe9+UdJWkw7Iez7JzIugYVkfE0IK/pWU45oSIGApcRPIgW7NExNciYlE6+98N1h3Q+vCAj1+XISTPk4wvUX4oyf3bzXUBcEvB/B/S16aapI+cZnUjHBFTI+KH6eyxJD1u1q67JCJ+34IYP0luB4r1kXQ9yefJ2pgTQQckqbuSMQlekPRHSRv02pn+in2y4BfzQenyz0t6Nt32PkndSxzuSWBQuu0F6b4WSPpGumwLSb+V9FK6fEy6fIakakk/BLqlcUxK161M/50i6aiCmG+XdIKkTpKulDRbSX/t/5HhZXmWtOMuScPTc3xR0jOSPpU+1Xo5MCaNZUwa+22Snk/LFuv9FOB4YFrDhRHxT2AuMCi92piVxvugpK3SWM7Tx+NITEmXnSrpp5IOAEYDV6Yx7VzwGoySdF/Ba1P3a7y576GkS9LXcoGkm6V6PfV+peAzMjwtn/V1Kaqx3jcj4nWgt6R/a87+LIP26G/bf+X9I3nCdF769yDJE+U903V9SJ5QrH24cGX67zeBb6fTnUj67ulD8sW+Rbr8W8AlRY53O3BCOn0i8BwwDPgjsAXJE84LgU+TfEneUrDtlum/M0jHD6iNqaBMbYzHAXek011IemTsBpwBfCddvhkwB6gqEufKgvO7DxiVzvcENk2nPwc8kE6fCvy0YPvvAyen071I+vXZosExqoC5BfOHAL9Jp3sDS4HBwHzg4HT55cBP0um3gM1qj9EwjsLXunA+fY//WvBe3Qic3ML3cOuC5XcBXyx4j25Jp0eQ9p/f2OvS4NyrgV808ZkdSJH++EmurI5v7/9TG9vfRtfFhBW1OpKqCAAkdQa+L2kE8C+SX8LbAX8r2GY2cFta9tcRMU/SwSTVEE+nPwq7kPySLuZKSd8h6fPldJK+YB6M5Fcwkn4FHETyS/lqST8i+ZL4QzPO63fAtZI2I6lKeDIiVkv6PLBXQR33liQdr/2lwfbdJM1Lz/9l4LGC8ndI2oWky4LOjRz/88BoSRem812BAem+am3Phv3eHCTpRZLX/ockHcX1ioiZ6fo7SBITJAlikqRfk/QjlEkkXTNMA74o6X7gKOC/SHqdzfoe1jpU0n8BmwNbkyTxh9N1k9PjPSmpp5J2lsZel8L45gBfy3o+Bd4BdmjBdtYEJ4KO6cvANsCwiPhISe+cXQsLpP+xR5B8gdwu6RrgPeCxiBiX4RgTIuL+2hlJI4sViog/p3XkRwLfkzQ9Ii7PchIRsUbSDOAIYAzJoCWQjNx0bkQ8WmIXqyNiqKTNSfqyGQ9cRzKYzRMRcZyShvUZjWwvkl+nrzR1DBq8tiRtBEfX7UTasontjyL5tf1F4NuS9myibENTgHNIqlnmRMQHabVO1vcQSV2Bn5Fcnb0haSL1z6dhHzVBI6+LpO2aEXtjupK8ptaG3EbQMW0JvJMmgUOBDcYvVjKm8d8j4hbgFyRD580CPiupts5/C0m7ZjzmH4BjJW0uaQuSap0/SNoBWBURvyTpGK9Yw+lH6ZVJMfeQdLpVe3UByZf6WbXbSNo1PWZRkYzcdh7wTX3cLXltt76nFhT9gKSKrNajwLm1deZKenht6M8k1RyNiogVwHtK22GArwAzlYyp0D8iniCpwtmSpFqtUMOYCs0keT2/zsdJsrnvYe2X/rtpW0LDO4lq23QOJOkFcwXZXpeW2hUoOpavtZwTQcc0CaiW9Efgq8CfipQ5BHgprcIYA1wbEctIvhgnS5pPUqWwW5YDRsQLJPXOz5O0GfwiIl4E9gSeT6toLgW+V2Tzm4H5ShuLG/h/JNUdv49kKENIEtci4AUltyD+nBJXv2ks80kGOfkx8IP03Au3ewLYo7axmOTKoXMa28J0vuF+/wm8VvvF24RTSKrT5pPcnXQ5SdvFL9P36UXguthwgJkpwIS0UXbnBsdeD/wG+EL6L819D9Pj3ULy5fsoSZVhoTXp63QTSRUgZHhdlNwI8Itix1TS++azwKck1Ug6PV3emeTGg0ruSvwTyb2PmuVM0nEk1XDfae9YKln6Ou4TEf+3vWPZ2LiNwCxnEfGgpEoeE/uTYlPg6vYOYmPkKwIzsw7ObQRmZh2cE4GZWQfnRGBm1sE5EZiZdXBOBGZmHdz/B7EDnZ0fremLAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#svm = SVC(C= 10.0,degree= 2, kernel= 'rbf', probability = True).fit(x_train_bowandtfidf,y_train)\n",
        "ypred = svm.predict(x_train_bowandtfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_train, ypred)))\n",
        "print('Precision Score:', (precision_score(y_train, ypred)))\n",
        "print('Recall Score:', (recall_score(y_train,ypred)))\n",
        "print('F1 Score:', (f1_score(y_train, ypred)))\n",
        "\n",
        "y_score = svm.predict_proba(x_train_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_train, y_score[:,1])))\n",
        "print(plot_roc_curve(svm, x_train_bowandtfidf, y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 140,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "drsO6DjHNgnL",
        "outputId": "ff6859ac-5f2b-4958-d9cd-b2335700b8bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.988255033557047\n",
            "Precision Score: 0.9464285714285714\n",
            "Recall Score: 0.7464788732394366\n",
            "F1 Score: 0.8346456692913385\n",
            "ROC Score: 0.8635927387270623\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A12DE413C0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAuQUlEQVR4nO3de3gU5fn/8fdNIAmQBIQEKyeJikVFpBLwUEXUWm21okULWNtqtdaK+v16oD9t+1Wg56qth1oUWi+xVfBUW6wWVBSstSgHEQFLixRLUEuKyEFIIOH+/TGzyybZZCcku2GTz+u6cjHHnXs24blnnmfmeczdERGR9qtDawcgIiKtS4lARKSdUyIQEWnnlAhERNo5JQIRkXauY2sH0FTFxcU+YMCA1g5DRCSrLFmy5L/uXpJsXdYlggEDBrB48eLWDkNEJKuY2bsNrVPVkIhIO6dEICLSzikRiIi0c0oEIiLtnBKBiEg7l7ZEYGYPmNlGM1vRwHozs7vNbI2ZLTezY9MVi4iINCyddwQPAmc1sv5zwMDw5wpgahpjERGRBqQtEbj7y8CHjWwyGnjIAwuB7mZ2ULriERHJVjV7nB8+s4oNH+1My+e3ZhtBH2B9wnx5uKweM7vCzBab2eKKioqMBCcisj9wd7771FtM/8u/WLA6PeVfVjQWu/s0dy9z97KSkqRvSIuItEk/m7uaWYvWc/Wph3HRcf3TcozWTAQbgH4J833DZSIiAkx7+R2mzn+Hi47rzw2fPTxtx2nNRDAb+Gr49NDxwBZ3f78V4xER2W88tmg9P3r275w95CC+P3owZpa2Y6Wt0zkzmwmMAorNrBy4FegE4O73Ac8CnwfWADuAS9MVi4hINpmz4gNu+v1yTh5YzC++NJScDulLApDGRODu41Osd2BCuo4vIpKN/rhsAxMfX84x/bpz38XDyO2Y/oqbrOuGWkSkLdpds4cfPvM2D766juEDDmD6V8vompeZIlqJQESklW3cWsmER5ayaN1mLv30AL7z+SPolJO5JlwlAhGRVrR43Ydc9fBStlbu5q5xQxk9NOnrVGmlRCAi0grcnRmvruMHz7xNnwM6M+PrIzjioKJWiUWJQEQkw3buquE7T73FU29s4PRBvfj52KF069yp1eJRIhARyaB3N33MN3+7hNX/2cb1ZxzO1aceRoc0Px6aSqREYGYdgGOA3sBOYIW7b0xnYCIibc1Lf9/I/8x6AzPjgUuGc+one7V2SECKRGBmhwL/D/gM8E+gAsgHDjezHcD9wAx335PuQEVEsoG789GO3VRsr2Lj1io2bqtk47Yq1lZs5/El5RzxiSLuu3gY/Xt2ae1Q41LdEfyAYJyAb4YvgMWZWS/gIuArwIz0hCcisn+ortnDpo931SrcY9MV26rYuK2KivBnV039a+POnXL40rB+TDr3KDrn5rTCGTSs0UTQ2NvBYdXQnS0dkIhIJlXurqlTuFfGC/WN8QK+kk0f76L25XDggC6dKCnMo1dhPocUd6WkKJjuVZgXLs+jV1E+BRl6OWxf7HNkZnaGuz/fksGIiLQEd2frzuq9hXvsqn1rVXx+47YqKrZWsa2qut7+OR2M4oJcehXm07tbPkP7daMkSeFeXJBLXsf96+p+XzQnRf0GSE/n2CIiSdTscTZtr2q0cN+4tYqK7VXsqq5fPZPfqUP8an3QJwoZObCEksTCvTCfXkV59OiS2+pP8mRSqsbi2Q2tAnq2fDgi0h5V7q4Jq2L21rfXrYuv2F7Fpu1V7ElSPdOtc6fwKj2PEaU94lfuJQmFe6/CPAryOqa1O+dsleqO4GTgYmB7neUGjEhLRCLSJrg7WyurqdiWUOeepHDfuLWSrZX1q2c6GBQXBIX7J7rlM6Rvt/iVe0lC4V5ckEd+p+yvnmlNqRLBQmCHuy+ou8LMVqcnJBHZn9XscTZ9vLcxtSJZ4b6tko1bq6hKUj2T17FDWIjnc1hJASce2jNeLVNSlBe/mu/ZNS/t/fBLINVTQ59rZN3Ilg9HRFpLVXVNrWqZigYK900f76ImSf1MUX5HehXlU1KQx7H9D6hV5554JV+Ur+qZ/c3++zyTiDSbu7O9qrrBZ95jhfvGbVVs2bm73v4dDHoW5FESVtEceVBRrTr3WB18SaGqZ7KZEoFIFtqzx/lwR+2XmyqSFO4V26rYubum3v65HTvEC/dDSrpy/CE99xbsCc/B9+iaS8cM9osvrUOJQGQ/sqt6T7wBNfGlpoqEwn3jtkr+uz159UxhXsd4PfvQft2TFu69CvMp6qzqGdlLiUAkA7ZXVSd5Y7UybGjdW2WzeUf96hkz6Nk1N/5C06BPFNZ6qalXQvXM/tZ1gWSHyInAzCa5+6SG5kXamz17nM07dtUr3IOG1tr18Tt2JameyekQf9Z9QM+uDB/Qo1b9e2y6p6pnJM2ackewJMW8SJuwu2ZPQn173Tr32o2t1UmqZwryOsarZAb36VavcI9dzXfv0knVM7JfiJwI3P3pxuZF9ncfV1UnLdxjV+6xdR9+vCvp/kH1TFDAH9arMGnh3qsojy65qnGV7JKqi4l7gCQvdAfc/doWj0ikCWJ9v9e/cq/9NM3GrZV8nKR6pmMHixfifQ/owrEHH5C0cC8uyKOTqmekjUp16bI4I1GI1FFds4f/bt/VYOEevNFaScX2KnbX1L9W6ZKbEy/Qj+xdxKhPltQr3HsV5tO9c6d21bmYSDKp3iyuNeCMmXVx9x3pDUnasp27amp3R5BQsMf6gq/YVsWHOxru+z1W535oSc/ahXvCUzRd9+O+30X2N1HHLD6BoNvpAqC/mR1DMGrZVekMTrKDu7Nl5+6E7ggS69/Dwn17w32/d+xg8c7F+h7QmU/1PyBp4V5ckEduR1XPiLS0qJdNdwJnArMB3P1NM1NfQ21c4tB8tQv32tU1DfX93rlTTrxB9YhPFDFyYF69599LCttf3+8i+5umPDW0vs6jbvVb3iQrxIbmS1a47x1wu4oPP07e93v3Lp3ihXhp8d6+3xMLd/X9LpI9oiaC9WZ2IuBm1gn4H+Dt9IUlTRUbmq/xwj2oj9+WpO/32NB8JYV7+37vVZhHSZ3CvaQwr00MzScie0VNBFcCdwF9gPeAucCEdAUle8X6fk98WzVZ4V6xLXXf74cfWMhJhxUHXQXXGZ6vR9dc9f0u0k5FSgTu/l/gy039cDM7iyCB5AC/dvef1FnfH5gBdA+3ucndn23qcbLR3qH59j45U28Ep20ND80X6/u9V2EeZQcfEO8Hfm/f78GTNYWqnhGRFKI+NXQIQYF+PMELZn8DrnP3tY3skwPcC5wBlAOLzGy2u69K2Ox7wGPuPtXMjgSeBQbsy4nsD9ydbVXVtfp9r0h4cib+/HuKvt9jT8sM7t0toefIvHinY+r7XURaUtSqoUcICvXzw/lxwEzguEb2GQGsiSULM5sFjAYSE4EDReF0N4Jqp/3Onj0ePD1Tt9/3rYnPwAcFf+Xu+tUzuR07xAv3w0oKOCHs+z1WZRMr7DU0n4i0hqiJoIu7/zZh/ndmNjHFPn2A9Qnz5dRPHJOA58zsGqAr8JlkH2RmVwBXAPTv3z9iyKklDs23d/zV2lfujfb9nt8xXsf+qX4H1CrcY9MlBer7XUT2b6n6GuoRTv7ZzG4CZhFcxY8lqMZprvHAg+5+R/jS2m/NbLC717qsdvdpwDSAsrKyBvs+SsXdufHx5by14SM2bqviowb7ft/7hMygTxQmLdzV97uItBWp7giWEBT8scvZbyasc+DmRvbdAPRLmO8bLkt0GXAWgLv/zczygWJgY4q49slHO3bz5NJyBvcp4pwhB8UL98TGVfX9LiLtTaq+hkqb8dmLgIFmVkqQAMYBF9XZ5t/A6cCDZnYEkA9UNOOYjYo10H7906V88di+6TqMiEhWacoIZYOBIwkKawDc/aGGtnf3ajO7muCdgxzgAXdfaWZTgMXuPhu4AZhuZtcR3GFc4p6sq7GWsbUySASF+Z3SdQgRkawT9fHRW4FRBIngWeBzwCtAg4kAIHwn4Nk6y25JmF4FfLpJETfD1p3BG7XdOisRiIjERK0Mv4CgCucDd78UOIbgcc+sErsjKOqsLopFRGKiJoKd4ZM81WZWRNCY2y/FPvudWBtBkaqGRETiol4aLzaz7sB0gieJthO8XZxVtoaJQFVDIiJ7Re1rKDYAzX1mNgcocvfl6Qur5e2u2cNji4P327ro+X8RkbhUL5Qd29g6d1/a8iGlx6J/fcg7FR+T36mD3vIVEUmQ6o7gjkbWOXBaC8aSVrtqgpeVH7hkeCtHIiKyf0n1QtmpmQokU9Rrp4hIbepLQUSknVMiEBFp55QIRETauUiJwAIXm9kt4Xx/MxuR3tBERCQTot4R/Ao4gWD8AIBtBCOWiYhIlov6ZvFx7n6smb0B4O6bzSw3jXGJiEiGRL0j2B0ORu8AZlYC1B+cV0REsk7URHA38BTQy8x+SNAF9Y/SFpWIiGRM1L6GHjazJQRdURtwnru/ndbIREQkI6IOTHM3MMvd1UAsItLGRK0aWgJ8z8zeMbPbzawsnUGJiEjmREoE7j7D3T8PDAdWAz81s3+mNTIREcmIpr5ZfBgwCDgY+HvLhyMiIpkW9c3in4V3AFOAFUCZu38hrZGJiEhGRH2h7B3gBHf/bzqDERGRzEs1Qtkgd/87sAjob2b9E9dn0whlIiKSXKo7guuBK0g+UllWjVAmIiLJpRqh7Ipw8nPuXpm4zszy0xaViIhkTNSnhl6NuExERLJMqjaCTwB9gM5m9imC7iUAioAuaY5NREQyIFUbwZnAJUBf4OcJy7cB30lTTCIikkGp2ghmADPMbIy7P5mhmEREJINSVQ1d7O6/AwaY2fV117v7z5PsJiIiWSRVY3HX8N8CoDDJT6PM7CwzW21ma8zspga2+ZKZrTKzlWb2SBNiFxGRFpCqauj+8N/JTf3gcESze4EzgHJgkZnNdvdVCdsMBG4GPh0Of9mrqccREZHmaUpfQ0Vm1snM5plZhZldnGK3EcAad1/r7ruAWcDoOtt8A7jX3TcDuPvGpp6AiIg0T9T3CD7r7luBc4B1BL2QTkyxTx9gfcJ8ebgs0eHA4Wb2VzNbaGZnJfsgM7vCzBab2eKKioqIIYuISBRRE0GsCuls4HF339JCx+8IDARGAeOB6WbWve5G7j7N3cvcvaykpKSFDi0iIhA9EfzJzP4ODAPmmVkJUJlinw1Av4T5vuGyROXAbHff7e7/Av5BkBhERCRDoo5QdhNwIsE4BLuBj6lf31/XImCgmZWaWS4wDphdZ5s/ENwNYGbFBFVFa6MGLyIizRd18PpOwMXASDMDWADc19g+7l5tZlcDc4Ec4AF3X2lmU4DF7j47XPdZM1sF1AAT3X3TPp+NiIg0WdSBaaYCnYBfhfNfCZdd3thO7v4s8GydZbckTDtBV9f1XlYTEZHMiJoIhrv7MQnzL5rZm+kISEREMitqY3GNmR0amzGzQwiqckREJMtFvSOYCLxkZmsJuqI+GLg0bVGJiEjGpEwE4aOiWwjeFI51AbHa3avSGZiIiGRGo1VDZnY5sBK4B1gGDHD35UoCIiJtR6o7gv8FjnL3irBd4GHqvwsgIiJZLFVj8S53rwBw97VAXvpDEhGRTEp1R9DXzO5uaN7dr01PWCIikimpEkHdHkaXpCsQERFpHVHGLBYRkTYs1VND081scAPruprZ183sy+kJTUREMiFV1dC9wC1mdjSwAqgA8gm6ii4CHiB4kkhERLJUqqqhZcCXzKwAKAMOAnYCb7v76vSHJyIi6Rapiwl33w7MT28oIiLSGqJ2OiciIm2UEoGISDvXpERgZl3SFYiIiLSOSInAzE4Mh5P8ezh/jJn9KsVuIiKSBaLeEfwCOBPYBODubwIj0xWUiIhkTuSqIXdfX2eRRigTEWkDoo5Qtt7MTgTczDoB/wO8nb6wREQkU6LeEVwJTAD6ABuAocBVaYpJREQyKOodwSfdvVafQmb2aeCvLR+SiIhkUtQ7gnsiLhMRkSzT6B2BmZ0AnAiUmNn1CauKgJx0BiYiIpmRqmooFygItytMWL4VuCBdQYmISOak6n10AbDAzB5093czFJOIiGRQ1MbiHWZ2G3AUwXgEALj7aWmJSkREMiZqY/HDBN1LlAKTgXXAojTFJCIiGRQ1EfR0998Au919gbt/HdDdgIhIGxC1amh3+O/7ZnY28B7QIz0hiYhIJkW9I/iBmXUDbgBuBH4N/G+qnczsLDNbbWZrzOymRrYbY2ZuZmUR4xERkRYSdajKP4WTW4BTIf5mcYPMLAe4FzgDKAcWmdlsd19VZ7tCgr6LXmta6CIi0hIavSMwsxwzG29mN5rZ4HDZOWb2KvDLFJ89Aljj7mvdfRcwCxidZLvvAz8FKpsevoiINFeqqqHfAJcDPYG7zex3wO3Az9z9Uyn27QMkdl1dHi6LM7NjgX7u/kxjH2RmV5jZYjNbXFFRkeKwIiLSFKmqhsqAIe6+x8zygQ+AQ919U3MPbGYdgJ8Dl6Ta1t2nAdMAysrKvLnHFhGRvVLdEexy9z0A7l4JrG1CEtgA9EuY7xsuiykEBgPzzWwdcDwwWw3GIiKZleqOYJCZLQ+nDTg0nDfA3X1II/suAgaaWSlBAhgHXBRb6e5bgOLYvJnNB25098VNPgsREdlnqRLBEfv6we5ebWZXA3MJeip9wN1XmtkUYLG7z97XzxYRkZaTqtO5ZnU05+7PAs/WWXZLA9uOas6xRERk30QevF5ERNomJQIRkXYuciIws85m9sl0BiMiIpkXKRGY2ReAZcCccH6omamxV0SkDYh6RzCJoMuIjwDcfRnB2AQiIpLloiaC3eFz/4n0hq+ISBsQdTyClWZ2EZBjZgOBa4FX0xeWiIhkStQ7gmsIxiuuAh4h6I76f9MUk4iIZFDUO4JB7v5d4LvpDEZERDIv6h3BHWb2tpl9PzYugYiItA2REoG7n0owMlkFcL+ZvWVm30trZCIikhGRXyhz9w/c/W7gSoJ3CpL2GSQiItkl6gtlR5jZJDN7C7iH4ImhvmmNTEREMiJqY/EDwKPAme7+XhrjERGRDIuUCNz9hHQHIiIiraPRRGBmj7n7l8IqocQ3iaOMUCYiIlkg1R3B/4T/npPuQEREpHU02ljs7u+Hk1e5+7uJP8BV6Q9PRETSLerjo2ckWfa5lgxERERaR6o2gm8RXPkfYmbLE1YVAn9NZ2AiIpIZqdoIHgH+DPwYuClh+TZ3/zBtUYmISMakSgTu7uvMbELdFWbWQ8lARCT7RbkjOAdYQvD4qCWsc+CQNMUlIiIZ0mgicPdzwn81LKWISBsVta+hT5tZ13D6YjP7uZn1T29oIiKSCVEfH50K7DCzY4AbgHeA36YtKhERyZioiaDa3R0YDfzS3e8leIRURESyXNTeR7eZ2c3AV4CTzawD0Cl9YYmISKZEvSMYSzBw/dfd/QOCsQhuS1tUIiKSMVGHqvwAeBjoZmbnAJXu/lBaIxMRkYyI+tTQl4DXgQuBLwGvmdkFEfY7y8xWm9kaM7spyfrrzWyVmS03s3lmdnBTT0BERJonahvBd4Hh7r4RwMxKgBeAJxrawcxygHsJOqwrBxaZ2Wx3X5Ww2RtAmbvvCPs1+hlBNZSIiGRI1DaCDrEkENoUYd8RwBp3X+vuu4BZBE8dxbn7S+6+I5xdiMZBFhHJuKh3BHPMbC4wM5wfCzybYp8+wPqE+XLguEa2v4ygg7t6zOwK4AqA/v31HpuISEuKOmbxRDP7InBSuGiauz/VUkGY2cVAGXBKA8efBkwDKCsr82TbiIjIvkk1HsFA4HbgUOAt4EZ33xDxszcA/RLm+4bL6h7jMwRtEKe4e1XEzxYRkRaSqp7/AeBPwBiCHkjvacJnLwIGmlmpmeUC44DZiRuY2aeA+4Fz67RBiIhIhqSqGip09+nh9GozWxr1g9292syuBuYCOcAD7r7SzKYAi919NsFLaQXA42YG8G93P7fJZyEiIvssVSLID6/aY+MQdE6cd/dGE4O7P0udRmV3vyVh+jNNjlhERFpUqkTwPvDzhPkPEuYdOC0dQYmISOakGpjm1EwFIiIirSPqC2UiItJGKRGIiLRzSgQiIu1c1N5HLRyr+JZwvr+ZjUhvaCIikglR7wh+BZwAjA/ntxH0LCoiIlkuaqdzx7n7sWb2BoC7bw7fFhYRkSwX9Y5gdzi+gEN8PII9aYtKREQyJmoiuBt4CuhlZj8EXgF+lLaoREQkY6J2Q/2wmS0BTifoXuI8d387rZGJiEhGREoEZtYf2AE8nbjM3f+drsBERCQzojYWP0PQPmBAPlAKrAaOSlNcIiKSIVGrho5OnDezY4Gr0hKRiIhk1D69WRx2P93Y+MMiIpIlorYRXJ8w2wE4FngvLRGJiEhGRW0jKEyYriZoM3iy5cMREZFMS5kIwhfJCt39xgzEIyIiGdZoG4GZdXT3GuDTGYpHREQyLNUdwesE7QHLzGw28DjwcWylu/8+jbG1KG/tAERE9lNR2wjygU0EYxTH3idwIHsSgQepoINZK0ciIrJ/SZUIeoVPDK1gbwKIyaqL7Jqwi7wcJQJpgt27d1NeXk5lZWVrhyISSX5+Pn379qVTp06R90mVCHKAAmongJgsSwThHYHGZJMmKC8vp7CwkAEDBmC6iJD9nLuzadMmysvLKS0tjbxfqkTwvrtPaV5o+4c9YdVQTgf9Z5boKisrlQQka5gZPXv2pKKiokn7pbo+bjN//bE7go5KBNJESgKSTfbl7zVVIjh930LZ/8SrhvSfWkSklkYTgbt/mKlA0i2WCFQ1JNnmP//5DxdddBGHHHIIw4YN44QTTuCpp55Kuu17773HBRdckHTdqFGjWLx4MQAPPPAARx99NEOGDGHw4MH88Y9/TFv869atY/DgwQ2uv/322xk0aBBDhw5l+PDhPPTQQ0yePJmbb7651nbLli3jiCOOSPoZF1xwAWvXrq21rZkxZ86cRuOYNGkSt99+e6OxNNeMGTMYOHAgAwcOZMaMGUm3WbZsGccffzxDhw6lrKyM119/Pb5u/vz5DB06lKOOOopTTjkFgF27djFy5Eiqq6ubHR/sY6dz2ahGj49KFnJ3zjvvPEaOHMnatWtZsmQJs2bNory8vN621dXV9O7dmyeeeKLRzywvL+eHP/whr7zyCsuXL2fhwoUMGTKk2bHuS6F033338fzzz/P666+zbNky5s2bh7szfvx4Hn300Vrbzpo1i/Hjx9f7jJUrV1JTU8MhhxwSXzZz5kxOOukkZs6c2exYmuPDDz9k8uTJvPbaa7z++utMnjyZzZs319vu29/+NrfeeivLli1jypQpfPvb3wbgo48+4qqrrmL27NmsXLmSxx9/HIDc3FxOP/30et/Rvor6HkHW26M7AmmmyU+vZNV7W1v0M4/sXcStX2h4WI8XX3yR3Nxcrrzyyviygw8+mGuuuQaABx98kN///vds376dmpoaZsyYwTnnnMOKFSvYuXMnl156KW+++SaDBg1i586dAGzcuJHCwkIKCgoAKCgoiE+/8847TJgwgYqKCrp06cL06dMZNGgQTz/9ND/4wQ/YtWsXPXv25OGHH+bAAw9k0qRJvPPOO6xdu5b+/ftz5513cuWVV8avzqdOnUrv3r2pqanhG9/4Bq+++ip9+vThj3/8I507d+ZHP/oR8+fPp6ioCICioiK+9rWvAXDAAQfw2muvcdxxQUfHjz32GHPnzq33HT388MOMHj06Pu/uPP744zz//POcfPLJVFZWkp+fn/J30Vgs+2ru3LmcccYZ9OjRA4AzzjiDOXPm1EtoZsbWrcHf1pYtW+jduzcAjzzyCF/84hfp378/AL169Yrvc95553HzzTfz5S9/uVkxQju8I1BjsWSTlStXcuyxxza6zdKlS3niiSdYsGBBreVTp06lS5cuvP3220yePJklS5YAcMwxx3DggQdSWlrKpZdeytNPxwce5IorruCee+5hyZIl3H777Vx1VTDsyEknncTChQt54403GDduHD/72c/i+6xatYoXXniBmTNncu2113LKKafw5ptvsnTpUo46Kkhy//znP5kwYQIrV66ke/fuPPnkk2zdupVt27bVupJPNH78eGbNmgXAwoUL6dGjBwMHDqy33V//+leGDRsWn3/11VcpLS3l0EMPZdSoUTzzzDONfn9AylgS3XbbbQwdOrTez7XXXltv2w0bNtCvX7/4fN++fdmwYUO97e68804mTpxIv379uPHGG/nxj38MwD/+8Q82b97MqFGjGDZsWK2qqsGDB7No0aKU8UbRbu4I9r5HoEQg+6axK/dMmTBhAq+88gq5ubnxQiDxijPRyy+/HC+chgwZEq/+ycnJYc6cOSxatIh58+Zx3XXXsWTJEm688UZeffVVLrzwwvhnVFVVAUF10tixY3n//ffZtWtXrWfUzz33XDp37gwEdzCxwionJ4du3bqxefNmSktLGTp0KADDhg1j3bp1Kc917NixnHjiidxxxx0NVgsBvP/++5SUlMTnZ86cybhx4wAYN24cDz30EGPGjGnwaZqmPmUzceJEJk6c2KR9Upk6dSq/+MUvGDNmDI899hiXXXYZL7zwAtXV1SxZsoR58+axc+dOTjjhBI4//ngOP/xwcnJyyM3NZdu2bRQWFqY+SCPSekdgZmeZ2WozW2NmNyVZn2dmj4brXzOzAemKJd5YrDYCySJHHXUUS5cujc/fe++9zJs3r9Zz4l27dm3y55oZI0aM4Oabb2bWrFk8+eST7Nmzh+7du7Ns2bL4z9tvvw3ANddcw9VXX81bb73F/fffX+tN6yjHz8vLi0/n5ORQXV1NUVERBQUFtRp5E/Xr14/S0lIWLFjAk08+ydixY5Nu17lz53g8NTU1PPnkk0yZMoUBAwZwzTXXMGfOHLZt20bPnj3r1c9/+OGHFBcXp4wlUVPuCPr06cP69evj8+Xl5fTp06fedjNmzOCLX/wiABdeeGG8sbhv376ceeaZdO3aleLiYkaOHMmbb74Z36+qqipStVcqaUsEYffV9wKfA44ExpvZkXU2uwzY7O6HAb8AfpqueHRHINnotNNOo7KykqlTp8aX7dixI9K+I0eO5JFHHgFgxYoVLF++HAieLEpMLsuWLePggw+mqKiI0tLSeIOku8cLnS1btsQLsIaefAE4/fTT47HW1NSwZcuWRmO8+eabmTBhQrx+fPv27bWqP8aPH891113HIYccQt++fZN+xhFHHMGaNWsAmDdvHkOGDGH9+vWsW7eOd999lzFjxvDUU09RUFDAQQcdxIsvvggESWDOnDmcdNJJkWKJmThxYq1kGfu5++6762175pln8txzz7F582Y2b97Mc889x5lnnllvu969e8er9l588cV4Fdjo0aN55ZVXqK6uZseOHbz22mvxJ6c2bdpEcXFxk7qSaEg67whGAGvcfa277wJmAaPrbDMaiP1VPQGcbml6e0dvFks2MjP+8Ic/sGDBAkpLSxkxYgRf+9rX+OlPU18zfetb32L79u0cccQR3HLLLfF69N27d3PjjTfGH5N89NFHueuuu4Cg4fU3v/kNxxxzDEcddVT8sdJJkyZx4YUXMmzYMIqLixs85l133cVLL73E0UcfzbBhw1i1alXKGE899VSGDx/O4MGDOfnkk+mQ0A/MhRdeyMqVKxusFgI4++yzmT9/PhBUC51//vm11o8ZMyb+9NBDDz3E97//fYYOHcppp53GrbfeyqGHHhopln3Ro0cP/u///o/hw4czfPhwbrnllng13uWXXx5/nHf69OnccMMNHHPMMXznO99h2rRpQJDkzjrrLIYMGcKIESO4/PLL44/AvvTSS5x99tnNii/O3dPyA1wA/Dph/ivAL+tsswLomzD/DlCc5LOuABYDi/v37+/74rmVH/i3frfYK3dX79P+0j6tWrWqtUOQFHbs2OHHHXecV1e3r//b559/vq9evTrpumR/t8Bib6C8zoqnhtx9mruXuXtZYqNQU5xx5IH86svDyOuY08LRiUhr6ty5M5MnT076NE5btWvXLs477zwOP/zwFvm8dD41tAHolzDfN1yWbJtyM+sIdCMY90BEJLJk9e5tWW5uLl/96ldb7PPSeUewCBhoZqVmlguMA2bX2WY2EHtj4wLgxfAWRmS/oT9JySb78veatkTg7tXA1cBc4G3gMXdfaWZTzOzccLPfAD3NbA1wPVDvEVOR1pSfn8+mTZuUDCQreDgeQVMfKbVs+wMvKyvzWEu7SLpphDLJNg2NUGZmS9y9LNk+7ebNYpF90alTpyaN9CSSjbLiqSEREUkfJQIRkXZOiUBEpJ3LusZiM6sA3t3H3YuB/7ZgONlA59w+6Jzbh+ac88HunvSN3KxLBM1hZosbajVvq3TO7YPOuX1I1zmrakhEpJ1TIhARaefaWyKY1toBtAKdc/ugc24f0nLO7aqNQERE6mtvdwQiIlKHEoGISDvXJhOBmZ1lZqvNbI2Z1evR1MzyzOzRcP1rZjagFcJsURHO+XozW2Vmy81snpkd3BpxtqRU55yw3RgzczPL+kcNo5yzmX0p/F2vNLNHMh1jS4vwt93fzF4yszfCv+/Pt0acLcXMHjCzjWa2ooH1ZmZ3h9/HcjM7ttkHbWjosmz9AXIIhrw8BMgF3gSOrLPNVcB94fQ44NHWjjsD53wq0CWc/lZ7OOdwu0LgZWAhUNbacWfg9zwQeAM4IJzv1dpxZ+CcpwHfCqePBNa1dtzNPOeRwLHAigbWfx74M2DA8cBrzT1mW7wjGAGscfe17r4LmAWMrrPNaGBGOP0EcLqZZfOo9inP2d1fcvcd4exCghHjslmU3zPA94GfAm2hH+ko5/wN4F533wzg7hszHGNLi3LODhSF092A9zIYX4tz95eBDxvZZDTwkAcWAt3N7KDmHLMtJoI+wPqE+fJwWdJtPBhAZwvQMyPRpUeUc050GcEVRTZLec7hLXM/d38mk4GlUZTf8+HA4Wb2VzNbaGZnZSy69IhyzpOAi82sHHgWuCYzobWapv5/T0njEbQzZnYxUAac0tqxpJOZdQB+DlzSyqFkWkeC6qFRBHd9L5vZ0e7+UWsGlWbjgQfd/Q4zOwH4rZkNdvc9rR1YtmiLdwQbgH4J833DZUm3MbOOBLeTmzISXXpEOWfM7DPAd4Fz3b0qQ7GlS6pzLgQGA/PNbB1BXersLG8wjvJ7Lgdmu/tud/8X8A+CxJCtopzzZcBjAO7+NyCfoHO2tirS//emaIuJYBEw0MxKzSyXoDF4dp1tZgNfC6cvAF70sBUmS6U8ZzP7FHA/QRLI9npjSHHO7r7F3YvdfYC7DyBoFznX3bN5nNMof9t/ILgbwMyKCaqK1mYwxpYW5Zz/DZwOYGZHECSCioxGmVmzga+GTw8dD2xx9/eb84FtrmrI3avN7GpgLsETBw+4+0ozmwIsdvfZwG8Ibh/XEDTKjGu9iJsv4jnfBhQAj4ft4v9293NbLehminjObUrEc54LfNbMVgE1wER3z9q73YjnfAMw3cyuI2g4viSbL+zMbCZBMi8O2z1uBToBuPt9BO0gnwfWADuAS5t9zCz+vkREpAW0xaohERFpAiUCEZF2TolARKSdUyIQEWnnlAhERNo5JYJ2wMxqzGxZws+ARrbd3gLHe9DM/hUea2n4tmdTP+PXZnZkOP2dOutebW6M4efEvpcVZva0mXVPsf3QfenZ0swOMrM/hdOjzGxLeNy3zezWffi8c2O9cJrZebHvKZyfEr442Czh7/CCFNvMb8oLeuG5/ynCdkl73zSz283stKjHk+iUCNqHne4+NOFnXQaOOdHdhwI3EbzI1iTufrm7rwpnv1Nn3YnNDw/Y+70MJnifZEKK7YcSPL/dVNcD0xPm/xJ+N2UEfeQ0qRthd5/t7j8JZ88j6HEztu4Wd39hH2LcnzwIJOsj6R6CvydpYUoE7ZCZFVgwJsFSM3vLzOr12hlexb6ccMV8crj8s2b2t3Dfx82sIMXhXgYOC/e9PvysFWb2v+Gyrmb2jJm9GS4fGy6fb2ZlZvYToHMYx8Phuu3hv7PM7OyEmB80swvMLMfMbjOzRRb01/7NCF/L3wg77jKzEeE5vmFmr5rZJ8O3WqcAY8NYxoaxP2Bmr4fbJuv9FGAMMKfuQnf/GFgCHBbebSwM433KzA4IY7nW9o4jMStcdomZ/dLMTgTOBW4LYzo04Ts4y8weT/hu4lfjTf0dmtkt4Xe5wsymmdXqqfcrCX8jI8Lto34vSTXU+6a7vwv0NLNPNOXzJILW6G9bP5n9IXjDdFn48xTBG+VF4bpigjcUYy8Xbg//vQH4bjidQ9B3TzFBwd41XP7/gFuSHO9B4IJw+kLgNWAY8BbQleAN55XApwgKyekJ+3YL/51POH5ALKaEbWIxng/MCKdzCXpk7AxcAXwvXJ4HLAZKk8S5PeH8HgfOCueLgI7h9GeAJ8PpS4BfJuz/I+DicLo7Qb8+XescoxRYkjA/CvhTON0TWAccBSwHTgmXTwHuDKffA/Jix6gbR+J3nTgf/o7/nfC7mgpcvI+/wx4Jy38LfCHhdzQ9nB5J2H9+Q99LnXMvA37dyN/sAJL0x09wZzWmtf9PtbWfNtfFhCS104OqCADMrBPwIzMbCewhuBI+EPggYZ9FwAPhtn9w92VmdgpBNcRfw4vCXIIr6WRuM7PvEfT5chlBXzBPeXAVjJn9HjiZ4Er5DjP7KUEh8ZcmnNefgbvMLI+gKuFld99pZp8FhiTUcXcj6HjtX3X272xmy8Lzfxt4PmH7GWY2kKDLgk4NHP+zwLlmdmM4nw/0Dz8r5iDq93tzspm9QfDd/4Sgo7ju7r4gXD+DIDFBkCAeNrM/EPQjFIkHXTPMAb5gZk8AZwPfJuh1NurvMOZUM/s20AXoQZDEnw7XzQyP97KZFVnQztLQ95IY32Lg8qjnk2Aj0Hsf9pNGKBG0T18GSoBh7r7bgt458xM3CP9jjyQoQB40s58Dm4Hn3X18hGNMdPcnYjNmdnqyjdz9H2Ed+eeBH5jZPHefEuUk3L3SzOYDZwJjCQYtgWDkpmvcfW6Kj9jp7kPNrAtBXzYTgLsJBrN5yd3Pt6BhfX4D+xvB1enqxo5Bne+WoI3gnPiHmHVrZP+zCa62vwB818yObmTbumYBVxNUsyx2921htU7U3yFmlg/8iuDubL2ZTaL2+dTto8Zp4HsxswObEHtD8gm+U2lBaiNon7oBG8MkcCpQb/xiC8Y0/o+7Twd+TTB03kLg02YWq/PvamaHRzzmX4DzzKyLmXUlqNb5i5n1Bna4++8IOsZL1nC6O7wzSeZRgk63YncXEBTq34rtY2aHh8dMyoOR264FbrC93ZLHuvW9JGHTbQRVZDFzgWtideYW9PBa1z8Iqjka5O5bgM0WtsMAXwEWWDCmQj93f4mgCqcbQbVaoroxJVpA8H1+g71Jsqm/w1ih/9+wLaHuk0SxNp2TCHrB3EK072VfHQ4kHctX9p0SQfv0MFBmZm8BXwX+nmSbUcCbYRXGWOAud68gKBhnmtlygiqFQVEO6O5LCeqdXydoM/i1u78BHA28HlbR3Ar8IMnu04DlFjYW1/EcQXXHCx4MZQhB4loFLLXgEcT7SXH3G8aynGCQk58BPw7PPXG/l4AjY43FBHcOncLYVobzdT/3Y+CdWMHbiK8RVKctJ3g6aQpB28Xvwt/TG8DdXn+AmVnAxLBR9tA6x64B/gR8LvyXpv4Ow+NNJyh85xJUGSaqDL+n+wiqACHC92LBgwC/TnZMC3rf/BvwSTMrN7PLwuWdCB48yOauxPdL6n1UJM3M7HyCarjvtXYs2Sz8Ho919/9r7VjaGrURiKSZuz9lZtk8Jvb+oiNwR2sH0RbpjkBEpJ1TG4GISDunRCAi0s4pEYiItHNKBCIi7ZwSgYhIO/f/AWbP4yRpMiZbAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "ypred_test = svm.predict(X_test_bowandtfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_test, ypred_test)))\n",
        "print('Precision Score:', (precision_score(y_test, ypred_test)))\n",
        "print('Recall Score:', (recall_score(y_test,ypred_test)))\n",
        "print('F1 Score:', (f1_score(y_test, ypred_test)))\n",
        "\n",
        "y_score = svm.predict_proba(X_test_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_test, y_score[:,1])))\n",
        "print(plot_roc_curve(svm, X_test_bowandtfidf, y_test))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 141,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'C': 0.1, 'degree': 2, 'kernel': 'rbf'}\n"
          ]
        }
      ],
      "source": [
        "print(svm.best_params_)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 142,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "ZDEz4QqDaFUe",
        "outputId": "bd3f0e08-0c44-4414-d2c1-1f4f3329bbd3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Most important features in SVM model')"
            ]
          },
          "execution_count": 142,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAc8AAAEWCAYAAAAASRzMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzrUlEQVR4nO3de7xd073//9dbom4JSnb7pZqzW8c1yu43W1oqmqpeTtoSp9GonJJSqRalPeHrW0qqx7eU30+LKvkqEUJVVKVUw6mQuITsLXcaitCW6kaREFTy+f4xxmqWZe+91tzZ97yfj8d6rLnGGnNc5krWZ48x55pDEYGZmZnVbqOeboCZmVlf4+BpZmZWkIOnmZlZQQ6eZmZmBTl4mpmZFeTgaWZmVpCDp1kHSBov6faebkdPkbSLpIWSVkr6Vk+3p7NIWiXpgz3djt5CUkj61xryjZL05+5oU2/h4Gl9kqQVkt6UNKQifUH+D1+/nuW3+6UREdMj4tPrU0dnkTRB0j2dWN5kSddUyXYKMDsiBkfEhetZ312SvrY+ZXSWiBgUEU90ZF9JR0v6Q/6D4jlJv5U0WNKpkua0kn9I/je8R/4MQ9IFFXkOzulTO9gl6yIOntaXPQl8ufRC0oeAzXuuOd1P0sAeqvpfgGU9VPfb9OAxKG/Dx4H/A3w5IgYDuwHX57evAfaV9IGK3Q4DlkTE0vz6ceBLFf05Eni061puHeXgaX3Z1cARZa+PBKaVZ5C0laRpklokPSXpdEkb5ff+VdLdkl6W9Lyk63N6aZSwKE/jjausuHK0l0cH35T0WB55/EDSjpLuk/SKpF9KelfOO0rSnyV9N9e7QtL4Gts8QdK9ki6Q9ALpC/pSYJ/c1pdyvs/lUfgrkv4kaXJZ+fW5vUdKejq34bT83meB7wLjcnmLWun7ncAngItznp0lbSLp/Fzec5IulbRZzv9uSbfk/vw9b++Q3zsbGFlW1sVl7RtYVuc/R6etHIPJVeofkut8SdKLkuaWjmcrffvnjIOkqZJ+KunW/Jk+IGnH1vYD9gbuj4gFABHxYkRcFRErI+LPwJ3AVyr2OYK3/3v9K7AE+EyufxtgX2BmG3WW/1s6RdLfJD0raYyk0ZIezf39bln+TST9WNIz+fFjSZuUvX9yLuMZSUdV1NXmMd4gRYQffvS5B7ACOBBYTvorfwDwZ9KIKID6nG8acDMwGKgn/RV/dH7vOuA00h+RmwL7lZUfwL+2U/8E4J6K/DcDWwLDgDeA3wMfBLYCHgaOzHlHAW8B/z+wCfBx4FVglxraPCHvewIwENissi1ldXwo921P4DlgTH6vPrf3/+b998rt3S2/Pxm4psrxvwv4WtnrC0hf8tvkdv8G+GF+b1vgi6RZgcHADcCv2ymr1L6BreVp4xi0V/8PSX9gbJwfIwG10a9/fu7AVOAFYESuZzrwizb2GwmsBr4PfAzYpOL98cBjZa93Ad4E6sr/PQGHA9fntG8ClwH/BUxto95R+Vickft2DNACXJuPw7Dcrg/k/GcB84D3AHXAfcAP8nufzf9O9gC2yGWUH4/2jvEo4M89/b3Qrd9BPd0AP/zoyIN1wfP0/OX4WeCO/CUX+Qt4QP6C2r1sv68Dd+XtacAUYIdWyu9I8PxY2etm4H+Vvf7/gB/n7dIX3hZl7/8S+F4NbZ4APN1eW9po74+BC/J2fW7vDmXvPwgclrcnUyB4AiIF/x3L3t8HeLKNfRuAv7dWVkX72gueT5e91279pIBxc3ufZ2ufOyl4Xl723mjgD+3s+2+kgPISsIr0x9GA/N7mwCvAvvn12cDNlZ8h6Q+B50h/cM0jBeJqwXN1WT2Dcx8+UvFvcUzefhwYXfbeZ4AVefsK4Jyy93YuHY8ajvEoNrDg6Wlb6+uuJv21PoGKKVtgCOmv8afK0p4C3pe3TyF9KTwoaVnlNFUHPFe2vbqV14PKXv89Il6taNf2NbQZ4E/VGiLpI5Jm56nSl4Fjc9nl/lq2/VpF+4qoIwWH5jw1+hLwu5yOpM0lXZanoF8B5gBbSxrQwfrg7ceg3fqB84A/ArdLekLSqQXqqfkYRcRtEfEF0sjsYNK/ya/l914jjbiPkCTSSLTy3ysRsRq4lfRH4bYRcW8NbXwhItbk7dX5ua1/e9vzzn9b25e996eK90qqHeMNjoOn9WkR8RTpwqHRwK8q3n4e+AdpKrdkKPCXvO9fI+KYiNieNLq7RDVclt9J3i1pi4p2PVOtzVnlUkitLY10LWmK7f0RsRVp2lI1tq3oUkvPk76gh0XE1vmxVUSUvrD/kzRN+ZGI2BLYP6eX2lNZX+mPivKLv/5HO21st/5I5x3/MyI+CBwEfEfSJwv2sWYRsTYifk86z7lH2VtXAV8CPsW6ac/WTCMds2pXPHfEM7zz39YzeftZ4P0V75VU+4w3OA6e1h8cDRxQMZIj/zX+S+BspZ8M/AvwHfKXkqRDSxeuAH8nfSGvza+fI52v7Erfl/QuSSOBzwM3VGtzG54DdlC+ICkbDLwYEa9LGkEandfqOaC+rYtqKkXEWtL50wskvQdA0vskfaasLauBl/JFMGe2Ut8Hy8prIf2x8B+SBuQZgbYu1Klav6TPK10cJuBlYA3rPudOofSTksPyxVHKx/zjpKnXkrmkKd0ppHOnb7ZR3N2kAHtRZ7Yxuw44XVKd0s+8zmDdv61fAhMk7S5pc8o+pxo+4w2Og6f1eRHxeEQ0tfH2CaSRzBOkc0rXks7tQLpC8gFJq0ijtBNj3W/8JgNX5SmqL3VBs/9KCtjPkC5EOTYi/lBDm1tzJ+lnI3+V9HxO+yZwlqSVpC/IXxZo2w35+QVJD9W4z/8iTY3Oy1Oz/00abUI637oZafQyjzTdV+4nwFilK3FLvxk9BjiZdMHOMNKFLR2tf6f8ehVwP3BJRMyusV+1+ntu82Okc5vXAOdFxPRShkgnB6eRRn7vmLItzxcRv4+IFzu5jZDOnzYBi0lX9j6U04iI20if1Z2kY3lnxb7tHeMNjvLJXjPrJpJGkS7I2aFKVjPrpTzyNDMzK8jB08zMrCBP25qZmRXkkaeZmVlBPX5DZeseQ4YMifr6+p5uhplZn9Lc3Px8RLzjZhAOnhuI+vp6mpra+jWHmZm1RtJTraV72tbMzKwgB08zM7OCHDzNzMwKcvA0MzMryMHTzMysIAdPMzOzghw8zczMCnLwNDMzK8g3SbDC6k+9taebYGZWsxXnfK7Ty/TI08zMrCAHTzMzs4IcPM3MzApy8DQzMyvIwdPMzKygfhU8JTVKurCn2wEgqUHS6Cp5Jkua1F1tMjOzztHjP1WRJEARsXZ9y4qIJuAdi1ZKGhgRb61v+QU1AI3Ab7u5XjMz62I9MvKUVC9puaRpwFLge5LmS1os6ftl+U6T9KikeyRdVxqlSbpLUmPeHiJpRd4eJemWvD1Z0tWS7gWullQn6cZcz3xJHyvLd5WkuZKekvTvkn4kaYmk30naOOcbLuluSc2SZknarqwt50p6MLd1pKR3AWcB4yQtlDSuncOxl6T7JT0m6ZhcpiSdJ2lpbse4nP5TSQfl7ZskXZG3j5J0divHeaKkJklNLS0tHf68zMzs7Xpy5LkTcCSwJTAWGAEImClpf+BV4DDSCG4g8BDQXLCO3YH9ImK1pGuBCyLiHklDgVnAbjnfjsAncv77gS9GxCmSbgI+J+lW4CLg4IhoycHsbOCovP/AiBiRp2nPjIgDJZ0BNEbE8VXauCfwUWALYEGua5/c772AIcB8SXOAucBIYCbwPmC7XMZI4BeVBUfEFGAKQGNjY9R4zMzMrIqeDJ5PRcQ8SecDnwYW5PRBpMA6GLgpIl4DkDSzA3XMjIjVeftAYPc0SwzAlpIG5e3bIuIfkpYAA4Df5fQlQD2wC7AHcEfefwDwbFk9v8rPzTl/ETfnNq6WNJv0R8R+wHURsQZ4TtLdwN6k4HmSpN2Bh4F35xHwPsC3CtZrZmYd1JPB89X8LOCHEXFZ+ZuSTmpn37dYN+W8aQ11kPN/NCJer6gH4A2AiFgr6R8RURqlrSUdIwHLImKfNup5Iz+vofgxrRwRtjlCjIi/SNoa+CwwB9gG+BKwKiJWFqzXzMw6qDdcbTsLOKo0CpT0PknvIQWHMZI2kzQY+ELZPiuA4Xl7bI313A6cUHohqaFAG5cDdZL2yftuLGlYlX1WkkbP1RwsaVNJ2wKjgPmkEeY4SQMk1QH7Aw/m/POAk0jHZy4wKT+bmVk36fHgGRG3A9cC9+dp0xnA4Ih4CLgeWATcRgoqJecD35C0gHROsBbfAhrzRUkPA8cWaOObpCB9rqRFwEJg3yq7zSZNE1e7YGhxzjsP+EFEPAPclNMXAXcCp0TEX3P+uaRzrH8knQfeBgdPM7NupXUzlL2bpMmk6cnze7otfVFjY2M0Nb3jVzwd4lVVzKwvWZ9VVSQ1R0RjZXqPjzzNzMz6mh6/SUKtImJyT7ehoyR9FTixIvneiDiuJ9qzvrpibTwzs76kzwTPviwirgSu7Ol2mJlZ5/C0rZmZWUEOnmZmZgV52tYK89W2ZtaXdMV1Gh55mpmZFeTgaWZmVpCDp5mZWUEOnmZmZgU5eJqZmRXUp4KnpMmSJnVSWZfndTHbyzNB0vZF9inYhqmSal0VxszMeokN9qcqEfG1GrJNAJYCzxTYx8zM+rleP/KUdJqkRyXdA+yS03aU9DtJzZLmSto1px8qaamkRZLm5LQBks7P6YslnZDT75LUmLdXSbpA0jJJv5dUl0eEjcD0vKzYZhX7fFnSklzuuWXtXSXp7NyGeZLeW6WLB0pqyn38fC5jU0lX5vIXSPpETr9V0p55e4GkM/L2WZKOaeXYTcxlN7W0tHT8QzAzs7fp1cFT0nDgMKABGA3snd+aApwQEcNJi0FfktPPAD4TEXsBB+W0iUA90BARewLTW6lqC6ApIoYBdwNnRsQMoAkYHxENEbG6rF3bA+cCB+S27S1pTFlZ83Ib5gDvCGoV6oERwOeASyVtChwHRER8CPgycFVOnwuMlLQV8BbwsVzGyFzX20TElIhojIjGurq6Ks0wM7Na9ergSQoKN0XEaxHxCjAT2JS0EPUNkhYClwHb5fz3AlPzKGxATjsQuCwi3gKIiBdbqWctaeFtgGuA/aq0a2/grohoyeVOB/bP770J3JK3m0nBsT2/jIi1EfEY8ASwa67/mtzePwBPATuTguf+pKB5KzBI0ubAByJieZV6zMysk/TFc54bAS9FREPlGxFxrKSPkEZxzXnk2hHrs0L4P2LdCuNrqH6MK+tqr+75pKnkJ4A7gCGkkW1zB9ppZmYd1NtHnnOAMfl842DgC8BrwJOSDgVQslfe3jEiHoiIM4AW4P2kIPN1SQNznm1aqWcjoHTV6+HAPXl7JTC4lfwPAh+XNETSANLU6t0d7OOhkjaStCPwQWA5aYQ5Prd3Z2AosDwi3gT+BBwK3J/zTaKVKVszM+s6vTp4RsRDpOnURcBtpJEXpMBytKRFwDLg4Jx+XukiHuC+vN/lwNPA4pz/8FaqehUYkfc7ADgrp08lnYdcKGmzsnY9C5wKzM51NEfEzR3s5tOkYHwbcGxEvE46h7uRpCW5/xMi4o2cfy7wt3wOdi6wQ342M7NuonUzjBsuSasiYlBPt6MrNTY2RlNTU6eU5VVVzKwvWZ9VVSQ1R0RjZXqvHnmamZn1Rn3xgqFO19WjTkmnkc5TlrshIs7uynrNzKxreNp2A9GZ07ZmZhsKT9uamZl1EgdPMzOzghw8zczMCvIFQ1aYf6piZr3N+vwcpSM88jQzMyvIwdPMzKwgB08zM7OCHDzNzMwK6lfBU1KjpAt7uh0Akhokja6SZ5Skfcte10l6QNICSSMl/VbS1q3sN1nSpLy9a75x/YK8MouZmXWxHg+eeUmxTmlHRDRFxLdaqaMnripuANoNnsAo0sLeJZ8ElkTEhyNibkSMjoiXqpQxBpiR93m8g201M7MCeiR4SqqXtFzSNGAp8D1J8yUtlvT9snynSXpU0j2Srisbbd0lqTFvD5G0Im+PknRL3p4s6WpJ9wJX51Hdjbme+ZI+VpbvKklzJT0l6d8l/SgvbfY7SRvnfMMl3S2pWdIsSduVteVcSQ/mto6U9C7Ssmbj8qhwXGvHADgW+HbOMxL4EXBwaQk0SSskDak8FsAuOW00cBLwDUmzO/VDMjOzNvXk7zx3Ao4EtiQtRD0CEDBT0v6kNTYPI43gBgIPAc0F69gd2C8iVku6FrggIu6RNBSYBeyW8+0IfCLnvx/4YkScIukm4HOSbgUuAg6OiJYcDM8Gjsr7D4yIETmYnRkRB0o6A2iMiONba1hErJB0KbAqIs4HqNxHEvl5eGvHIiJ+W1lGOUkTgYkAQ4cOLXjozMysLT0ZPJ+KiHmSzgc+DSzI6YNIgXUwcFNEvAYgaWYH6piZF40GOBDYvRSQgC0llVZTuS0i/pEXnx4A/C6nLwHqSSO9PYA78v4DgGfL6vlVfm7O+TvbSDpwLCJiCjAF0o3hu6BdZmYbpJ4Mnq/mZwE/jIjLyt+UdFI7+77FuinnTWuog5z/oxHxekU9AG8ARMRaSf+IdUvNrCUdIwHLImKfNup5Iz+vwXdtMjPr93r8giHS9OlRpVGgpPdJeg8wBxiTz/0NBr5Qts8KYHjeHltjPbcDJ5ReSGoo0MblQJ2kffK+G0saVmWflaTR8/rmgfaPhZmZdbMeD54RcTtwLXB/njadAQyOiIeA64FFwG3A/LLdziddJLMAGFJjVd8CGvNFSQ+TLtaptY1vkoL0uZIWAQt5+1WyrZlNmiZu9YKh7DfAIWUXDLVVf3vHwszMulmfWQxb0mTauDDGquvMxbB9Y3gz62266sbw8mLYZmZmnaPPXNwSEZN7ug0dJemrwIkVyfdGxHE90R4zM1s/fSZ49mURcSVwZU+3o7N097p5Zma9jadtzczMCnLwNDMzK8jB08zMrCCf87TC/FMVM+ttuvtaDI88zczMCnLwNDMzK8jB08zMrCAHTzMzs4IcPM3MzArqU8FT0mRJkzqprMsl7V4lzwRJ2xfZp2AbxpSXJ2nXvMLKAkk7Srqvjf2mShqbt0dKWpb326yz2mZmZm3rU8GzM0XE1yLi4SrZJgD/DJ417lPEGGD3itczIuLDEfF4RFRb9gxgPGkx8YaIWN2JbTMzszb0+uAp6TRJj0q6B9glp+0o6XeSmiXNlbRrTj9U0lJJiyTNyWkDJJ2f0xdLOiGn3yWpMW+vknRBHsH9XlJdHtk1AtNLo7qKfb4saUku99yy9q6SdHZuwzxJ722jX/sCBwHn5fKPA04irVM6u1RWfpakiyUtl/TfwHty+teALwE/kDS9lTomSmqS1NTS0rKen4SZmZX06uApaThwGNAAjAb2zm9NAU6IiOHAJOCSnH4G8JmI2IsUmAAmAvVAQ0TsCbwjyABbAE0RMQy4GzgzImYATcD4ylFdnso9Fzggt21vSWPKypqX2zAHOKa1vkXEfcBM4ORc/k+BS4ELIuITFdkPIf3hsDtwBHkh7oi4vKyM8a3UMSUiGiOisa6urrVmmJlZB/Tq4AmMBG6KiNci4hVSoNiUFDxukLQQuAzYLue/F5gq6RhgQE47ELgsIt4CiIgXW6lnLXB93r4G2K9Ku/YG7oqIllzudGD//N6bwC15u5kUuNfX/sB1EbEmIp4B7uyEMs3MrIP64u35NgJeioiGyjci4lhJHwE+BzTnkWtHxHq07x8RUdp/DX3zGJuZWTt6+8hzDjAmn28cDHwBeA14UtKh8M/zgXvl7R0j4oGIOANoAd4P3AF8XdLAnGebVurZCBibtw8H7snbK4HBreR/EPi4pCGSBgBfJk33FtVW+ZXmAOPy+dvtgMppXTMz60a9OnhGxEOk6dRFwG3A/PzWeOBoSYuAZcDBOf280kU8wH15v8uBp4HFOf/hrVT1KjAi73cAcFZOnwpcWvkzkIh4FjgVmJ3raI6ImzvQxV8AJ5d+mtJOvpuAx4CHgWnA/R2oy8zMOonWzTBuuCStiohBPd2OrtTY2BhNTU2dUpZXVTGz3qarVlWR1BwRjZXpvXrkaWZm1hv5Yhagq0edkk4DDq1IviEizu7KertKd6+bZ2bW2zh4doMcJPtkoDQzs3fytK2ZmVlBDp5mZmYFedrWCvPVtsX5PLFZ/+KRp5mZWUEOnmZmZgU5eJqZmRXk4GlmZlaQg6eZmVlBDp4FSDpP0rL8fKykI3L6VEljq+3fSnkT8sLapdcjc/kLJb1P0ow29rtLUmPePlTSI5Jmd7RfZmZWjH+qUkaSSDfLX9tGlonANhGxppOqnAAsBZ7Jr8cDP4yIa/LrWgLy0cAxEXFP1ZxmZtYpahp5SjpC0mJJiyRdLale0p057feShuZ8UyX9TNI8SU9IGiXpijwymlpW3ipJF+RR1u8l1eX0YyTNz/XcKGnzsnIvlHRfLndsTp8maUxZudMlHUwr8ijv5jxqe0zSmTm9XtJySdNIgez9eWS5NC9vNi7nmwkMIi2yPU7SZEmTWqlnuKS7JTVLmpXX32ytPWOBRmB6HmmeAHwJ+EHuR31eIo28nukv8nG8Cdgsp58B7Af8XNJ5rdQxUVKTpKaWlpY2P18zMyumavCUNAw4HTggIvYCTgQuAq6KiD2B6cCFZbu8G9gH+DYwE7gAGAZ8SFJDzrMF0BQRw0iLSJ+Z038VEXvneh4hjapKtiMFis8D5+S0n5NGb0jaCtgXaO8X/COALwJ7AoeWpj6BnYBLcnsagQZgL+BA0hqh20XEQcDqiGiIiOvbOFYb52MzNiKGA1fQxj1tI2IG0ASMz2VelI/XyRExviL7N4DXImI30rEanss4q6yMk1upY0pENEZEY11dXTuHxczMiqhl5HkAaQWQ5wEi4kVScLw2v381KaiV/CbSIqFLgOciYkmeBl0G1Oc8a0mLXANcU7b/HpLmSlpCmsIcVlburyNibUQ8DLw3t+VuYKc8cv0ycGNEvNVOX+6IiBciYjXwq7J6n4qIeXl7P+C6iFgTEc+RgvveVY5RyS7AHsAdkhaS/ujYocZ927M/6TgREYuBxZ1QppmZdVBXnPN8Iz+vLdsuvW6rvtKK3FOBMRGxSNIEYFQr5QKobHsa8B/AYcBXq7StcuXv0utXq+xXKwHLImKfTirPzMx6oVpGnneSpji3BZC0DXAfKVhBGiHO7UC9pYthDgdKF7sMBp7N05+VU5dtmQqcBJBHpe35lKRtJG0GjAHubSXPXGCcpAF5RLs/8GCNbVkO1EnaB9I0bp72bstKUp+rmUM6TkjagzTtbGZmPaTqyDMilkk6G7hb0hpgAXACcKWkk4EWqo/4Kr0KjJB0OvA3YFxO/x7wQC7zAWoILBHxnKRHgF/XUO+DwI2kqdRrIqJJUn1FnptI09KLSCPTUyLirzWUTUS8mS8EujCfgx0I/Jg0Zd2aqcClklbnOtvyM9LxfoR0Lri5lvaYmVnXUDo92c2VSqsiYlAnlbU56fzq/4yIl9vJNwFojIjjO6PevqaxsTGampo6pSyvqlKcV1Ux65skNUdEY2V6n75JgqQDSSOxi9oLnGZmZp2pR0aeXUnSZ4BzK5KfjIhDeqI9AJJ+CnysIvknEXFld7WhM0eeZmYbirZGnv3uDkMRMQuY1dPtKBcRx/V0G8zMrPP06WlbMzOznuDgaWZmVlC/m7a1ruerbYvz1bZm/YtHnmZmZgU5eJqZmRXk4GlmZlaQg6eZmVlBDp5mZmYFOXgWIOk8Scvy87GSjsjpU/MN4YuWd1fZgtxmZtZH+KcqZSSJdMvCtW1kmQhsExFrurFZZmbWy9Q08pR0hKTFkhZJulpSvaQ7c9rvJQ3N+aZK+pmkeZKekDRK0hWSHpE0tay8VZIuyKO43+d1M5F0jKT5uZ4b84oppXIvlHRfLndsTp8maUxZudMlHdxGHyZIujmP9h6TdGZOr5e0XNI0YCnw/jyyXCppiaRxOd9MYBDQLGmcpMmSJrVSz3BJd0tqljRL0nZVDu9XJC3M9Y3IZWwj6df5+M6TtGdOXyJpayUvlI18p0n6VCttmSipSVJTS0tLlWaYmVmtqgbPvJjz6cABEbEXcCJwEXBVROwJTAcuLNvl3aS1Kb8NzAQuAIYBH5LUkPNsATRFxDDgbuDMnP6riNg71/MIcHRZudsB+wGfB87JaT8HJuR2bgXsC7T3C/4RwBdJi0kfWjZluhNwSW5PI9AA7AUcCJwnabuIOAhYHRENEXF9G8dq43xsxkbEcOAK4Ox22gOweUQ0AN/M+QG+DyzIx/e7wLScfi/pBvPDgCeAkTl9H9IC5W8TEVMiojEiGuvq6qo0w8zMalXLyPMA4IaIeB4gIl4kfVlfm9+/mhTUSn4TaamWJcBzEbEkT4MuA+pznrVAKQBdU7b/HpLmSloCjCcFiZJfR8TaiHgYeG9uy93ATnnk+mXgxoh4q52+3BERL0TEauBXZfU+FRHz8vZ+wHURsSYiniMF972rHKOSXYA9gDskLST90bFDlX2uy32ZA2wpaevchqtz+p3AtpK2BOYC++fHz0h/kLwP+HtEvFpjG83MbD11xTnPN/Lz2rLt0uu26iutizYVGBMRi/Li1aNaKRdAZdvTgP8ADgO+WqVtleuvlV53VuARsCwi9imwT1ttas0c4DhgKHAacAgwlhRUzcysm9Qy8ryTNMW5LaTzcaQpwsPy++Mp/uW9EelLH+Bw4J68PRh4Nk9/jq+xrKnASQB5VNqeT+XziZsBY0jToJXmAuMkDcgj2v2BB2tsy3KgTtI+kKZx87R3e0rnVPcDXs6Les8l91/SKOD5iHglIv4EDAF2iognSMdtEimomplZN6k68oyIZZLOBu6WtAZYAJwAXCnpZKCF6iO+Sq8CIySdDvyNHECA7wEP5DIfIAXTau17TtIjwK9rqPdB4EbSVOo1EdEkqb4iz02kaelFpFHgKRHx1xrKJiLezBczXZjPwQ4Efkyasm7L65IWABsDR+W0ycAVkhYDrwFHluV/ABiQt+cCP2TdHx9mZtYNlE5PdnOl0qqIGNRJZW1OOr/6P/Oora18E4DGiDi+M+rtaxobG6OpqalTyvKqKsV5VRWzvklSc0S84/f4ffomCZIOJF2Ve1F7gdPMzKwz9cjIsytJ+gxwbkXykxFxSE+0B0DST0k/MSn3k4i4srva0JkjTzOzDUVbI89+d4ehiJgFzOrpdpSLiON6ug1mZtZ5+vS0rZmZWU9w8DQzMyuo303bWtfryNW2vtrUzPoTjzzNzMwKcvA0MzMryMHTzMysIAdPMzOzghw8zczMCnLwbIOkrSV9M29vL2lG3m6QNLos3wRJF3eg/MmSJnVei83MrLs4eLZta+CbABHxTESUllBrAEa3sY+ZmW0AHDzbdg6wo6SFkm6QtFTSu4CzSOt9LpQ0rnwHSXWSbpQ0Pz8q72dbaS9J90t6TNIxuQxJOi/Xt6RUh6SfSjoob98k6Yq8fVReMu4dJE2U1CSpqaWlZf2OhpmZ/ZNvktC2U4E9IqIhr/l5S16v8wzKljbLS52V/AS4ICLukTSUdI/d3dqpY0/go8AWwAJJt5LWEm0A9iItfD1f0hzS2p0jgZnA+4DtchkjgV+0VnhETAGmQLoxfKHem5lZmxw8O9eBwO6SSq+3lDQoIla1kf/miFgNrJY0GxgB7AdcFxFrgOck3Q3sTQqeJ0naHXgYeLek7UjB9ltd1yUzM6vk4Nm5NgI+GhGv15i/cjTY5ugwIv4iaWvgs8AcYBvgS8CqiFjZgbaamVkH+Zxn21YCgwukA9wOnFB6IamhSh0HS9pU0rbAKGA+aYQ5TtIASXXA/sCDOf884CRS8JwLTMrPZmbWjRw82xARLwD3SloKnFf21mzS1Ow7LhgiTZ82Slos6WHg2CrVLM7lzQN+EBHPADfl9EXAncApEfHXnH8uMDAi/gg8RBp9OniamXUzRfg6kg1BY2NjNDU1dUpZXlXFzDYUkpojorEy3SNPMzOzgnzBUBeT9FXgxIrkeyPiuJ5oT2fwKNLMNnQOnl0sIq4EruzpdpiZWefxtK2ZmVlBDp5mZmYFOXiamZkV5OBpZmZWkIOnmZlZQQ6eZmZmBTl4mpmZFeTgaWZmVlC/CZ6S6vNN3LuyjvMkLcvPx0o6IqdPkLR9B8obk9fnLL3eNd9wfoGkHSXd18Z+UyWNzdsjc5sWStqso30zM7Pa+Q5DxUwEtskLVZebACwFnilY3hjgFtLi1qXXMyLiv/LrfWsoYzzww4i4pmDdZmbWQf1m5JkNlDRd0iOSZkjaXNJwSXdLapY0S9J2AJLuknSupAclPSppZE4fkEeW8/PSYl/P6TOBQUCzpHGSJkualEeAjcD00uhP0jmSHs77n99aQyXtCxwEnJf3O460Vuc3JM3OeVblZ0m6WNJySf8NvCenf420IPYPJE3vqoNqZmZv199GnrsAR0fEvZKuAI4DDgEOjoiWvP7m2cBROf/AiBghaTRwJnAgcDTwckTsLWkT0pqet0fEQZJWRUQDgKTJABExQ9LxwKSIaMoLWx8C7BoRIWnr1hoaEfflgHxLRMzIZdYBqyKiMuAekvu2O/Be0kj1ioi4XNJ+5WWUkzSRNFpm6NChxY6kmZm1qb8Fzz9FxL15+xrgu8AewB2SAAYAz5bl/1V+bgbq8/angT1L5xSBrYCdgCdrbMPLwOvAzyXdQpqWXV/7A9fl6eJnJN1Zy04RMQWYAmk9z05oh5mZ0f+CZ2WAWAksi4h92sj/Rn5ew7pjIeCEiJjVoQZEvCVpBPBJYCxwPHBAR8oyM7Peqb+d8xwqqRQoDwfmAXWlNEkbSxpWpYxZpPOOG+d9dpa0RZV9VgKDc/5BwFYR8Vvg28BetexXxRxgXD4fux3wiRr2MTOzLtLfgudy4DhJjwDvBi4ijf7OlbQIWEj1K1gvJ51TfCj/9OUyqo/QpwKXSlpICoa3SFoM3AN8p539fgGcXPppSjv5bgIey+2aBtxfpT1mZtaFFOFTYRuCxsbGaGpq6ulmmJn1KZKaI6KxMr2/jTzNzMy6XH+7YKhXknQacGhF8g0RcXZPtMfMzNaPg2c3yEHSgdLMrJ/wtK2ZmVlBDp5mZmYFOXiamZkV5OBpZmZWkIOnmZlZQQ6eZmZmBTl4mpmZFeTgaWZmVlC/CZ6S6vON3LuyjvMkLcvPx0o6IqdPkLR9B8qbWrZuqJmZ9RG+w1AxE4Ft8qLU5SYAS4Fnur1FZmbW7frNyDMbKGm6pEckzZC0uaThku6W1CxpVl4PE0l3STpX0oOSHpU0MqcPyCPL+ZIWS/p6Tp8JDAKaJY2TNFnSpDxybASmS1ooaTNJ50h6OO9/fpU2HyipKbfh87muTSVdKWlJXq7sEzn9Vkl75u0Fks7I22dJOqYLjqeZmbWivwXPXYBLImI34BXgOPKanhExHLiCt99jdmBEjABOAs7MaUcDL0fE3sDewDGSPhARBwGrI6IhIq4vFRARM4AmYHxENACbA4cAwyJiT+C/qrS5HhgBfI60Juimud0RER8CvgxcldPnAiMlbQW8BXwslzGStGD220iamANzU0tLS5VmmJlZrfpb8PxTRNybt68BPgPsAdyRF6o+HdihLP+v8nMzKYgBfBo4Iud/ANgW2KlAG14GXgd+Lunfgdeq5P9lRKyNiMeAJ4Bdgf1y+4mIPwBPATuTguf+pKB5KzBI0ubAByJieWXBETElIhojorGurq5AF8zMrD397Zxn5creK4FlEbFPG/nfyM9rWHcsBJwQEbM61ICItySNAD4JjAWOBw4o0Ob2ViefT5oifgK4AxgCHEMK/mZm1k3628hzqKRSoDwcmAfUldIkbSxpWJUyZgHfkLRx3mdnSVtU2WclMDjnHwRsFRG/Bb4N7FVl30MlbSRpR+CDwHLSCHN8qX5gKLA8It4E/kRaG/T+nG8SrUzZmplZ1+lvI8/lwHGSrgAeJp3vnAVcmM8TDgR+DCxrp4zLSVO4D0kS0AKMqVLvVNL5ytXAvwE353OUAr5TZd+ngQeBLYFjI+J1SZcAP5O0hHRuc0JElEbJc4FPRsRqSXNJ09Bzq9RhZmadSBHtzRJaf9HY2BhNTU093Qwzsz5FUnNENFam97dpWzMzsy7X36ZteyVJp5HOU5a7ISLObi2/mZn1bg6e3SAHSQdKM7N+wtO2ZmZmBTl4mpmZFeTgaWZmVpDPeVpV9afe+rbXK875XA+1xMysd/DI08zMrCAHTzMzs4IcPM3MzApy8DQzMyvIwdPMzKwgB882SNpa0jfz9vaSZuTtBkmjy/JNkHRxB8ofJWnfstd1kh6QtEDSSEm/lbR1K/tNljQpb+8qaWHeZ8cOdNPMzDrAwbNtWwPfBIiIZyJibE5vAEa3sU8Ro4B9y15/ElgSER+OiLkRMToiXqpSxhhgRt7n8U5ok5mZ1cDBs23nADvmkd0NkpZKehdwFjAup48r3yGPHm+UND8/PtZawZLqgWOBb+dyRgI/Ag7OrzeTtELSkJz/NEmPSroH2CWnjQZOIi3cPbuNeiZKapLU1NLS0hnHxMzM8E0S2nMqsEdENORgd0tEvCnpDKAxIo6HNG1bts9PgAsi4h5JQ0kLce9WWXBErJB0KbAqIs7P5VSWS34eDhxGGvEOBB4CmiPit5VltFLPFGAKpPU81+dgmJnZOg6enetAYPdS4AO2lDQoIlatR5kjgZsi4jUASTPXs41mZraeHDw710bARyPi9Z5uiJmZdR2f82zbSmBwgXSA24ETSi8kNXSg/EpzgDH5POhg4As17GNmZl3IwbMNEfECcK+kpcB5ZW/NJk3NvuOCIeBbQKOkxZIeJl0U1JbfAIeUXTDUVjseAq4HFgG3AfM70B0zM+tEivB1JBuCxsbGaGpq6tC+XlXFzDZUkpojorEy3SNPMzOzgnzBUBeT9FXgxIrkeyPiuJ5oT0d4pGlm9nYOnl0sIq4EruzpdpiZWefxtK2ZmVlBDp5mZmYFOXiamZkV5OBpZmZWkIOnmZlZQQ6eZmZmBTl4mpmZFeTgaWZmVpDvbbuBkNQCPNVJxQ0Bnu+ksnqzDaGfG0Ifwf3sT7q7j/8SEXWViQ6eVpikptZulNzfbAj93BD6CO5nf9Jb+uhpWzMzs4IcPM3MzApy8LSOmNLTDegmG0I/N4Q+gvvZn/SKPvqcp5mZWUEeeZqZmRXk4GlmZlaQg6ch6bOSlkv6o6RTW3l/E0nX5/cfkFRf9t7/zunLJX2m1jK7Wxf1cYWkJZIWSmrqpq60q6P9lLStpNmSVkm6uGKf4bmff5R0oSR1U3da1UV9vCuXuTA/3tNN3WnTevTzU5Ka82fWLOmAsn161WeZ29QV/ez6zzMi/NiAH8AA4HHgg8C7gEXA7hV5vglcmrcPA67P27vn/JsAH8jlDKilzL7ex/zeCmBIT3+GndTPLYD9gGOBiyv2eRD4KCDgNuDf+mEf7wIae/oz7KR+fhjYPm/vAfylN36WXdzPLv88PfK0EcAfI+KJiHgT+AVwcEWeg4Gr8vYM4JP5L9aDgV9ExBsR8STwx1xeLWV2p67oY2/U4X5GxKsRcQ/wenlmSdsBW0bEvEjfStOAMV3ZiSo6vY+91Pr0c0FEPJPTlwGb5dFbb/ssoQv62S2txtO2Bu8D/lT2+s85rdU8EfEW8DKwbTv71lJmd+qKPgIEcHueMprYBe0uan362V6Zf65SZnfqij6WXJmn+L7XC6YzO6ufXwQeiog36H2fJXRNP0u69PMc2NkFmm1A9ouIv+TzKXdI+kNEzOnpRlmHjM+f5WDgRuArpJFZnyVpGHAu8OmebktXaqOfXf55euRpfwHeX/Z6h5zWah5JA4GtgBfa2beWMrtTV/SRiCg9/w24iZ6fzl2ffrZX5g5VyuxOXdHH8s9yJXAtffyzlLQD6d/kERHxeFn+3vRZQtf0s1s+TwdPmw/sJOkDkt5FOiE/syLPTODIvD0WuDOfM5kJHJbPp3wA2Il0QUItZXanTu+jpC3yX7VI2oL0V+/SbuhLe9ann62KiGeBVyR9NE99HQHc3PlNr1mn91HSQElD8vbGwOfpw5+lpK2BW4FTI+LeUuZe+FlCF/Sz2z7P7ryyyo/e+QBGA4+Srno7LaedBRyUtzcFbiBdLPMg8MGyfU/L+y2n7Mq91srsT30kXR24KD+W9YY+dkI/VwAvAqtI5552z+mNpC+fx4GLyXcm6y99JF2F2wwszp/lT8hXVPfFfgKnA68CC8se7+mNn2VX9LO7Pk/fns/MzKwgT9uamZkV5OBpZmZWkIOnmZlZQQ6eZmZmBTl4mpmZFeTgadYLSVqTby22VNINkjbvxrobJI0umk/SQa2titHBNqzqjHIK1Fcv6fDurNP6NgdPs95pdUQ0RMQewJuklUCqyndgWV8NpN/eFcoXETMj4pxOqL9b5WNWDzh4Ws38O0+zXkjSqogYlLePBfYETgYuIi2/tDEwOSJuljQB+HdgEGmJpytJq2VsQboj0vmk5Z6+ArwBjI6IFyXdBUyKiKZ8R5YmYGfSj9E3I90W7YfAk6Qfmm8KrAa+mtMq821GWgbq+Lzm4hXAEKAF+GpEPC1pKvAK6cf6/wM4JSJmtNV/SaOA7wMvAR8CfgksAU7M9Y2JiMdzua/ncrcEvhMRt0jaFPhZTn8rp89u5ZhtAuyW+3UV6ZZvV+djCHB8RNyX2zMZeD5/Ds3Af0RESNo7H6ct8nH+JPAacA4wKtfx04i4rLK/1gf19N0l/PDDj3c+gFX5eSDpFmrfAP4P6YsaYGvSXVm2ACaQ7pazTX5vAimwDQbqSKtQHJvfuwA4KW/fRV7zkBTkVpTtf3FZW7YEBubtA4Eb28j3z9fAb4Aj8/ZRwK/z9lTS3WI2It3d549V+j+KFDi3IwWfvwDfz++dCPy4rNzf5XJ3ysdjU+A/gStynl2Bp3N65TEbBdxSVv/mwKZ5eyegqSzfy6R7sG4E3E9aI/RdwBPA3uXHDJgInJ7TNiH9gfKBnv735cf6P7yqilnvtJmkhXl7LvBz4D7gIEmTcvqmwNC8fUdEvFi2/+xIN8VeKellUjCDNGrbs2BbtgKukrQTaRm2jWvYZx/SyA7SCO5HZe/9OiLWAg9Lem8NZc2PdF9WJD0O3J7TlwCfKMv3y1zuY5KeIAXL/UijdSLiD5KeIo2u4Z3HrNzGwMWSGoA1ZfsAPBgRf87tWUia8n0ZeDYi5ue6XsnvfxrYU9LYvO9WpGD8ZA39tl7MwdOsd1odEQ3lCflm3l+MiOUV6R8h3eOzXPm6hmvLXq9l3f/7t1h33cOm7bTlB6RgfEiejr2rti60qbxttayzWEtfIAV22nldqfKYlfs28BywF+kYlS+gXd6eNbT/PSrghIiYVaUt1sf4giGzvmMWcEJpYV9JH17P8lYAw/P22LL0laQp35KtWLdM1IR28pW7j7RCBsB40ui5qx0qaSNJO5Ju3L881zseQNLOpJH68lb2ba3Pz+aR7FdI50XbsxzYLp/3RNLgfCHSLOAbeXUPJO2cV+GxPs7B06zv+AFpOnGxpGX59fo4n/TFvoB0zrNkNrB7/qnMONKU6w9zvoHt5Ct3AvBVSYtJwefE9WxrLZ4mrbpxG+kc7+vAJcBGkpYA1wMTIuKNVvZdDKyRtEjSt/N+R0paRJr+bW+USkS8CYwDLsr73EEazV8OPAw8JGkpcBme8esXfLWtmfV5+WrbW6KVK3fNuoJHnmZmZgV55GlmZlaQR55mZmYFOXiamZkV5OBpZmZWkIOnmZlZQQ6eZmZmBf0/2ZtyTNw7K24AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "from sklearn.inspection import permutation_importance\n",
        "svm = SVC(C= 10.0,degree= 2, kernel= 'rbf', probability = True).fit(x_train_bowandtfidf,y_train)\n",
        "perm_importance = permutation_importance(svm, X_test_bowandtfidf, y_test)\n",
        "\n",
        "feature_names = ['title_bow', 'company_profile_bow', 'desciption_bow', 'requirement_bow', 'benefits_bow', 'title_tfidf', 'company_profile_tfidf', 'desciption_tfidf', 'requirement_tfidf', 'benefits_tfidf']\n",
        "features = np.array(feature_names)\n",
        "\n",
        "sorted_idx = perm_importance.importances_mean.argsort()\n",
        "plt.barh(features[sorted_idx], perm_importance.importances_mean[sorted_idx])\n",
        "plt.xlabel(\"Permutation Importance\")\n",
        "plt.title(\"Most important features in SVM model\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 143,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VtsklJHUfH4Y",
        "outputId": "c4b1e2a3-4655-4595-f184-65a5b456b2a4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n",
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\naive_bayes.py:591: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "#Most important ngrams\n",
        "desc_nb_tfidf = MultinomialNB(alpha= 0, fit_prior = True)\n",
        "requirement_nb_tfidf = MultinomialNB(alpha= 0, fit_prior = True)\n",
        "\n",
        "desc_result = desc_nb_tfidf.fit(X_train_description_tfidf, y_train)\n",
        "requirement_result = requirement_nb_tfidf.fit(X_train_requirements_tfidf, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 144,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V0IDamauedXx",
        "outputId": "8ed577a4-f735-4659-fa42-57da5c8eb53d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['work' 'amp' 'We' 'position' 'skills' 'home' 'data' 'time' 'experience'\n",
            " 'customer']\n"
          ]
        }
      ],
      "source": [
        "pos_class_prob_sorted_description = desc_nb_tfidf.feature_log_prob_[1, :].argsort()[::-1]\n",
        "print(np.take(tfidf_vect_description.get_feature_names_out(), pos_class_prob_sorted_description[:10]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 145,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ex-ne64feMCZ",
        "outputId": "d474f9e0-aa85-46ca-85ef-03f93cb8d59e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['assure quality standards consistentlyattained'\n",
            " 'France Germany Central amp' 'evaluating maintaining'\n",
            " 'help run Holiday Market' 'France headquarters UK Germany'\n",
            " 'FranceABOUT MENTIONMention changes way' 'DevelopmentWe believe'\n",
            " 'Products Inc leading' 'effectivelySupport mentor junior members'\n",
            " 'background assist Kenya Nakuru']\n"
          ]
        }
      ],
      "source": [
        "pos_class_prob_sorted_description = requirement_nb_tfidf.feature_log_prob_[1, :].argsort()[::-1]\n",
        "print(np.take(tfidf_vect_description.get_feature_names_out(), pos_class_prob_sorted_description[:10]))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3iPW7LHi38nX"
      },
      "source": [
        "## Logistic Regression\n",
        "{'C': 0.1, 'penalty': 'l2', 'random_state': 1234, 'solver': 'liblinear'}  <br> The above is the determined best parameters.\n",
        "<br> I did not choose this model as it underperformed in terms of ROC-AUC and F1 compared to the support vector machine."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 146,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "oRIoMHRcN970",
        "outputId": "74947052-7dd8-47c4-d459-21f31b856f40"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 20 candidates, totalling 60 fits\n",
            "[CV 1/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=0.994 total time=   1.7s\n",
            "[CV 2/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.7s\n",
            "[CV 3/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.5s\n",
            "[CV 2/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.8s\n",
            "[CV 3/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.3s\n",
            "[CV 1/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.3s\n",
            "[CV 2/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.7s\n",
            "[CV 3/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.0s\n",
            "[CV 1/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.1s\n",
            "[CV 2/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.1s\n",
            "[CV 3/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.6s\n",
            "[CV 2/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.0s\n",
            "[CV 3/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.6s\n",
            "[CV 1/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.6s\n",
            "[CV 2/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.6s\n",
            "[CV 3/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.5s\n",
            "[CV 1/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.9s\n",
            "[CV 2/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.2s\n",
            "[CV 3/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.1s\n",
            "[CV 1/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.8s\n",
            "[CV 2/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.8s\n",
            "[CV 3/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.8s\n",
            "Fitting 3 folds for each of 20 candidates, totalling 60 fits\n",
            "[CV 1/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=0.994 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=0.994 total time=   1.6s\n",
            "[CV 2/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.7s\n",
            "[CV 3/3] END C=0.1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=0.1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.5s\n",
            "[CV 2/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.8s\n",
            "[CV 3/3] END C=1, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.3s\n",
            "[CV 1/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=1, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=10, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.3s\n",
            "[CV 2/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.8s\n",
            "[CV 3/3] END C=10, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.0s\n",
            "[CV 1/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=10, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.1s\n",
            "[CV 2/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.1s\n",
            "[CV 3/3] END C=10, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.1s\n",
            "[CV 1/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=100, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.6s\n",
            "[CV 2/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.0s\n",
            "[CV 3/3] END C=100, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   1.7s\n",
            "[CV 1/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=100, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.6s\n",
            "[CV 2/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.6s\n",
            "[CV 3/3] END C=100, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.5s\n",
            "[CV 1/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=200, penalty=l1, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=0.998 total time=   1.9s\n",
            "[CV 2/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.2s\n",
            "[CV 3/3] END C=200, penalty=l1, random_state=1234, solver=saga;, score=1.000 total time=   2.1s\n",
            "[CV 1/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=0.998 total time=   0.0s\n",
            "[CV 2/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 3/3] END C=200, penalty=l2, random_state=1234, solver=liblinear;, score=1.000 total time=   0.0s\n",
            "[CV 1/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=0.998 total time=   0.8s\n",
            "[CV 2/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.8s\n",
            "[CV 3/3] END C=200, penalty=l2, random_state=1234, solver=saga;, score=1.000 total time=   0.8s\n",
            "Accuracy Score: 0.9996271439224459\n",
            "Precision Score: 0.9949811794228356\n",
            "Recall Score: 0.9974842767295597\n",
            "F1 Score: 0.9962311557788944\n",
            "ROC Score: 0.9980785890109583\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A13E34EFB0>\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmJElEQVR4nO3de5xVdb3/8ddbRPECqIBGXAQFS0QlmWPeMpQ08t5PEykvdFQ6mWlSnkOn8pZ1NG8dSys0j5gEqZ2UTPNogZoJCYgoIIGIMoiKYCqRqfD5/bHWjHtm9sxec1l7nNnv5+MxD/Za67vW+qy9h/ns9f2u7/eriMDMzCrXFu0dgJmZtS8nAjOzCudEYGZW4ZwIzMwqnBOBmVmF27K9A2iu3r17x6BBg9o7DDOzDmXevHmvRUSfYts6XCIYNGgQc+fObe8wzMw6FEkvNLbNVUNmZhXOicDMrMI5EZiZVTgnAjOzCudEYGZW4XJLBJJukfSqpGca2S5J10taLmmhpP3yisXMzBqX5x3BrcCYJrZ/Bhia/kwAfpJjLGZm1ojc+hFExCOSBjVR5HjgtkjGwZ4taQdJfSNiTV4xbd4c1B90u/4w3MUG5a4/UnfDozQsU0yxMlmOVTym5sddrGBbnb9YuazX2zCmYucrfZzi56tfJr/PLsv5i8WQ7bNr6fUWK5PhM2/h9RbTks+uLX/nsrx3mX7nMpy/+LFLf1Ab39nEsy+/yTvvbS5yxPeN3nMX9h2wQ5NlWqI9O5T1A1YVLFen6xokAkkTSO4aGDhwYItONnvFOr5w8xw2bfb8C2b2wSU1vm3nHt06XSLILCImA5MBqqqqWvSX/MX1G9m0ORhbNYB+O25TZ1v9973YB6GmPp2m9qt39OJlSh+r/nEaO1bDMkX2y3CchmWynb/BqgznL1Ysy/VmOU7RY2X+DJShTAvO30i5tjh/sfO19HeuWKlsn0ELf+cy/A5kWZXl/FnPl+V6i8fU/M+g/07b0m+HbRoWLIP2TASrgQEFy/3Tdbn66ugh9N9x27xPY2bWYbTn46MzgNPTp4cOAN7Is33AzMyKy+2OQNI0YBTQW1I1cDHQFSAifgrcBxwFLAc2Al/MKxYzM2tcnk8NjSuxPYCv5HV+MzPLxj2LzcwqnBOBmVmFcyIwM6twTgRmZhWuchKBOxSbmRVVOYkglaWHsJlZJcn0+KikLYB9gQ8D/wCeiYhX8wzMzMzKo8lEIGl34D+ATwHLgLVAN2APSRuBnwFTIqLpIfPMzOwDq9QdweUk8wR8KeqNwSppZ+DzwGnAlHzCaztFh4I1M7OmE0FTvYPTqqEftnVAeXMLgZlZXS1uLJZ0RFsGYmZm7aM1Tw39vM2iMDOzdlOqsXhGY5uAXm0fTn6yTqtnZlZpSjUWfwI4FdhQb72A/XOJKGfuRmBmVlepRDAb2BgRD9ffIGlpPiGZmVk5lXpq6DNNbDu07cMxM7Nyq5ghJtxEYGZWXMUkghpyTwIzszoqLhGYmVldTgRmZhUucyKQdElTyx907kdgZlZcc+4I5pVY7hDcj8DMrK7MiSAiftvUspmZdUylhpj4EU08eRkR57V5RGZmVlalehbPLUsUZeD5CMzMiivVs7jOhDOSto2IjfmGlC83EZiZ1ZWpjUDSgZIWA8+my/tKujHXyMzMrCyyNhb/EPg0sA4gIp4CPNaQmVkn0JynhlbVW7WpjWPJlfsRmJkVV6qxuMYqSQcBIakrcD6wJL+wcuRGAjOzOrLeEfwb8BWgH/ASMCJdNjOzDi5TIoiI1yLiCxGxS0T0iYhTI2Jdqf0kjZG0VNJySZOKbB8oaaakJyUtlHRUSy7CzMxaLutTQ7tJ+q2ktZJelXSPpN1K7NMFuAH4DDAMGCdpWL1i3wbuiIiPAacAuT2J5CYCM7PislYN/RK4A+gLfBi4E5hWYp/9geURsSIi3gGmA8fXKxNAj/R1T5Jqp1x5PgIzs7qyJoJtI+IXEfFe+nM70K3EPv2AwieNqtN1hS4BTpVUDdwHfLXYgSRNkDRX0ty1a9dmDNnMzLJoMhFI2knSTsD9kiZJGiRpV0n/TvKHu7XGAbdGRH/gKOAXkhrEFBGTI6IqIqr69OnTBqc1M7MapR4fnUdSfVNTn/Klgm0BfLOJfVcDAwqW+6frCp0JjAGIiMcldQN6A6+WiKv53JHAzKyoUmMNDW7FsZ8AhkoaTJIATgE+X6/Mi8Bo4FZJe5JUN+Va9+P5CMzM6sraoQxJw0me/qltG4iI2xorHxHvSToXeADoAtwSEYskXQbMjYgZwNeBmyRdQHKHMT7CX93NzMopUyKQdDEwiiQR3EfySOifgEYTAUBE3Ee9toSIuKjg9WLg4GZFbGZmbSrrU0MnkVThvBwRXwT2JXncs8PwbYaZWXFZE8E/ImIz8J6kHiSNuQNK7POB5CYCM7O6srYRzJW0A3ATyZNEG4DH8wrKzMzKJ1MiiIhz0pc/lfR7oEdELMwvLDMzK5dSk9fv19S2iJjf9iHlw88imZkVV+qO4JomtgVweBvGUhZyRwIzszpKdSg7rFyBmJlZ+8g8VaWZmXVOTgRmZhWuYhKBR64wMysu6wxlknSqpIvS5YGS9s83tHy4qdjMrK6sdwQ3AgeSzB8A8BbJNJRmZtbBZe1Z/PGI2E/SkwAR8bqkrXKMy8zMyiTrHcG76WT0ASCpD7A5t6hy4BYCM7PisiaC64HfADtL+h7JENTfzy2qHLk/mZlZXVnHGpoqaR7JUNQCToiIJblGZmZmZZF1YprrgekR4QZiM7NOJmvV0Dzg25Kek3S1pKo8g8qDuxGYmRWXKRFExJSIOAr4F2ApcKWkZblGlhO5J4GZWR3N7Vk8BPgosCvwbNuHY2Zm5Za1Z/EP0juAy4BngKqIODbXyMzMrCyydih7DjgwIl7LM5g8uYnAzKy4UjOUfTQingWeAAZKGli4vSPNUFbLTQRmZnWUuiOYCEyg+ExlHXKGMjMzq6vUDGUT0pefiYi3C7dJ6pZbVGZmVjZZnxr6c8Z1ZmbWwZRqI/gQ0A/YRtLHeL+GvQewbc6xmZlZGZRqI/g0MB7oD1xbsP4t4D9zisnMzMqoVBvBFGCKpBMj4tdlisnMzMqoVNXQqRFxOzBI0sT62yPi2iK7mZlZB1KqsXi79N/tge5FfpokaYykpZKWS5rUSJmTJS2WtEjSL5sRu5mZtYFSVUM/S/+9tLkHTmc0uwE4AqgGnpA0IyIWF5QZCnwTODid/nLn5p7HzMxapzljDfWQ1FXSHyStlXRqid32B5ZHxIqIeAeYDhxfr8zZwA0R8TpARLza3AswM7PWydqP4MiIeBM4BlhJMgrphSX26QesKliuTtcV2gPYQ9JjkmZLGlPsQJImSJorae7atWszhmxmZllkTQQ1VUhHA3dGxBttdP4tgaHAKGAccJOkHeoXiojJEVEVEVV9+vRpo1ObmRlkTwT3SnoWGAn8QVIf4O0S+6wGBhQs90/XFaoGZkTEuxHxPPBXksRgZmZlknWGsknAQSTzELwL/J2G9f31PQEMlTRY0lbAKcCMemXuJrkbQFJvkqqiFVmDNzOz1ss6eX1X4FTgUEkADwM/bWqfiHhP0rnAA0AX4JaIWCTpMmBuRMxItx0paTGwCbgwIta1+GrMzKzZsk5M8xOgK3Bjunxauu6spnaKiPuA++qtu6jgdZAMdd2gs5qZmZVH1kTwLxGxb8HyHyU9lUdAZmZWXlkbizdJ2r1mQdJuJFU5ZmbWwWW9I7gQmClpBclQ1LsCX8wtKjMzK5uSiSB9VPQNkp7CNUNALI2If+YZmJmZlUeTVUOSzgIWAT8CFgCDImKhk4CZWedR6o7ga8BeEbE2bReYSsO+AGZm1oGVaix+JyLWAkTECmDr/EMyM7NyKnVH0F/S9Y0tR8R5+YRlZmblUioR1B9hdF5egZiZWfvIMmexmZl1YqWeGrpJ0vBGtm0n6V8lfSGf0MzMrBxKVQ3dAFwkaW/gGWAt0I1kqOgewC0kTxKZmVkHVapqaAFwsqTtgSqgL/APYElELM0/PDMzy1umISYiYgMwK99QzMysPWQddM7MzDopJwIzswrXrEQgadu8AjEzs/aRKRFIOiidTvLZdHlfSTeW2M3MzDqArHcE1wGfBtYBRMRTwKF5BWVmZuWTuWooIlbVW+UZyszMOoGsM5StknQQEJK6AucDS/ILy8zMyiXrHcG/AV8B+gGrgRHAOTnFZGZmZZT1juAjEVFnTCFJBwOPtX1IZmZWTlnvCH6UcZ2ZmXUwTd4RSDoQOAjoI2liwaYeQJc8AzMzs/IoVTW0FbB9Wq57wfo3gZPyCsrMzMqn1OijDwMPS7o1Il4oU0xmZlZGWRuLN0q6CtiLZD4CACLi8FyiMjOzssnaWDyVZHiJwcClwErgiZxiMjOzMsqaCHpFxM+BdyPi4Yj4V6BD3Q1ERHuHYGb2gZS1aujd9N81ko4GXgJ2yiekfEntHYGZ2QdL1juCyyX1BL4OfAO4GfhaqZ0kjZG0VNJySZOaKHeipJBUlTEeMzNrI1mnqrw3ffkGcBjU9ixulKQuwA3AEUA18ISkGRGxuF657iRjF81pXuhmZtYWmrwjkNRF0jhJ35A0PF13jKQ/Az8ucez9geURsSIi3gGmA8cXKfdd4Erg7eaHb2ZmrVWqaujnwFlAL+B6SbcDVwM/iIiPldi3H1A4dHV1uq6WpP2AARHxu6YOJGmCpLmS5q5du7bEac3MrDlKVQ1VAftExGZJ3YCXgd0jYl1rTyxpC+BaYHypshExGZgMUFVV5cd/zMzaUKk7gnciYjNARLwNrGhGElgNDChY7p+uq9EdGA7MkrQSOACY4QZjM7PyKnVH8FFJC9PXAnZPlwVEROzTxL5PAEMlDSZJAKcAn6/ZGBFvAL1rliXNAr4REXObfRVmZtZipRLBni09cES8J+lc4AGSkUpviYhFki4D5kbEjJYe28zM2k6pQedaNdBcRNwH3Fdv3UWNlB3VmnOZmVnLZJ683szMOqeKSwQeYcLMrK7MiUDSNpI+kmcwZmZWfpkSgaRjgQXA79PlEZLc2Gtm1glkvSO4hGTIiL8BRMQCkrkJzMysg8uaCN5Nn/sv5B6+ZmadQNb5CBZJ+jzQRdJQ4Dzgz/mFZWZm5ZL1juCrJPMV/xP4Jclw1F/LKSYzMyujrHcEH42IbwHfyjMYMzMrv6x3BNdIWiLpuzXzEpiZWeeQKRFExGEkM5OtBX4m6WlJ3841MjMzK4vMHcoi4uWIuB74N5I+BUXHDPqgCj/jZGZWVNYOZXtKukTS08CPSJ4Y6p9rZDmRPMiEmVmhrI3FtwC/Aj4dES/lGI+ZmZVZpkQQEQfmHYiZmbWPJhOBpDsi4uS0Sqiwlj3LDGVmZtYBlLojOD/995i8AzEzs/bRZGNxRKxJX54TES8U/gDn5B+emZnlLevjo0cUWfeZtgzEzMzaR6k2gi+TfPPfTdLCgk3dgcfyDMzMzMqjVBvBL4H7gf8CJhWsfysi1ucWlZmZlU2pRBARsVLSV+pvkLSTk4GZWceX5Y7gGGAeyeOjhd1yA9gtp7jMzKxMmkwEEXFM+m+Hn5YyPKGamVlRWccaOljSdunrUyVdK2lgvqHlwyMNmZnVlfXx0Z8AGyXtC3wdeA74RW5RmZlZ2WRNBO9FRADHAz+OiBtIHiE1M7MOLuvoo29J+iZwGvAJSVsAXfMLy8zMyiXrHcFYkonr/zUiXiaZi+Cq3KIyM7OyyTpV5cvAVKCnpGOAtyPitlwjMzOzssj61NDJwF+AzwEnA3MknZRhvzGSlkpaLmlSke0TJS2WtFDSHyTt2twLMDOz1snaRvAt4F8i4lUASX2Ah4C7GttBUhfgBpIB66qBJyTNiIjFBcWeBKoiYmM6rtEPSKqhzMysTLK2EWxRkwRS6zLsuz+wPCJWRMQ7wHSSp45qRcTMiNiYLs6mg86DbGbWkWW9I/i9pAeAaenyWOC+Evv0A1YVLFcDH2+i/JkkA9w1IGkCMAFg4MAO2Y/NzOwDK+ucxRdK+n/AIemqyRHxm7YKQtKpQBXwyUbOPxmYDFBVVdWisSLCI0yYmRVVaj6CocDVwO7A08A3ImJ1xmOvBgYULPdP19U/x6dI2iA+GRH/zHjsFpPHmDAzq6NUPf8twL3AiSQjkP6oGcd+AhgqabCkrYBTgBmFBSR9DPgZcFy9NggzMyuTUlVD3SPipvT1Uknzsx44It6TdC7wANAFuCUiFkm6DJgbETNIOqVtD9yp5Kv6ixFxXLOvwszMWqxUIuiWfmuvqVDZpnA5IppMDBFxH/UalSPiooLXn2p2xGZm1qZKJYI1wLUFyy8XLAdweB5BmZlZ+ZSamOawcgViZmbtI2uHMjMz66ScCMzMKpwTgZlZhcs6+qjSuYovSpcHSto/39DMzKwcst4R3AgcCIxLl98iGVnUzMw6uKyDzn08IvaT9CRARLye9hY2M7MOLusdwbvp/AIBtfMRbM4tKjMzK5usieB64DfAzpK+B/wJ+H5uUZmZWdlkHYZ6qqR5wGiS4SVOiIgluUZmZmZlkSkRSBoIbAR+W7guIl7MKzAzMyuPrI3FvyNpHxDQDRgMLAX2yikuMzMrk6xVQ3sXLkvaDzgnl4jMzKysWtSzOB1+uqn5h83MrIPI2kYwsWBxC2A/4KVcIjIzs7LK2kbQveD1eyRtBr9u+3DMzKzcSiaCtCNZ94j4RhniMTOzMmuyjUDSlhGxCTi4TPGYmVmZlboj+AtJe8ACSTOAO4G/12yMiP/NMTYzMyuDrG0E3YB1JHMU1/QnCMCJwMysgyuVCHZOnxh6hvcTQI3ILSozazPvvvsu1dXVvP322+0dipVBt27d6N+/P127ds28T6lE0AXYnroJoIYTgVkHUF1dTffu3Rk0aBBSsf/K1llEBOvWraO6uprBgwdn3q9UIlgTEZe1LjQza09vv/22k0CFkESvXr1Yu3Zts/Yr1bPYvzlmnYCTQOVoyWddKhGMblkoZmbWUTSZCCJifbkCMbPO63vf+x577bUX++yzDyNGjGDOnDlceumlfPOb36xTbsGCBey5554AbNiwgS996UvsvvvujBw5klGjRjFnzpwGx44IDj/8cN58883adXfffTeSePbZZ2vXzZo1i2OOOabOvuPHj+euu+4Ckkb1SZMmMXToUPbbbz8OPPBA7r///lZd97p16zjssMPYfvvtOffccxstt379eo444giGDh3KEUccweuvv157beeddx5Dhgxhn332Yf78+QCsXbuWMWPGtCq2Qi0adM7MLKvHH3+ce++9l/nz57Nw4UIeeughBgwYwLhx4/jVr35Vp+z06dMZN24cAGeddRY77bQTy5YtY968efzP//wPr732WoPj33fffey777706NGjdt20adM45JBDmDZtWuY4v/Od77BmzRqeeeYZ5s+fz913381bb73VwqtOdOvWje9+97tcffXVTZa74oorGD16NMuWLWP06NFcccUVANx///0sW7aMZcuWMXnyZL785S8D0KdPH/r27ctjjz3WqvhqZO1HYGadwKW/XcTil94sXbAZhn24Bxcf2/jUJGvWrKF3795svfXWAPTu3bt224477sicOXP4+MeTwYzvuOMOHnjgAZ577jnmzJnD1KlT2WKL5Pvq4MGDiz4JM3XqVCZMmFC7vGHDBv70pz8xc+ZMjj32WC699NKS17Bx40Zuuukmnn/++do4d9llF04++eQM70DjtttuOw455BCWL1/eZLl77rmHWbNmAXDGGWcwatQorrzySu655x5OP/10JHHAAQfwt7/9jTVr1tC3b19OOOEEpk6dysEHt37gB98RmFmujjzySFatWsUee+zBOeecw8MPP1y7bdy4cUyfPh2A2bNns9NOOzF06FAWLVrEiBEj6NKlS8njP/bYY4wcObJ2+Z577mHMmDHsscce9OrVi3nz5pU8xvLlyxk4cGCdu4rGXHDBBYwYMaLBT823+JZ45ZVX6Nu3LwAf+tCHeOWVVwBYvXo1AwYMqC3Xv39/Vq9eDUBVVRWPPvpoi89ZyHcEZhWkqW/uedl+++2ZN28ejz76KDNnzmTs2LFcccUVjB8/nrFjx3LQQQdxzTXX1KkWao7169fTvfv7AyRPmzaN888/H4BTTjmFadOmMXLkyEafpmnuUzbXXXdds2NsDkmZYtp555156aW2mQ0g10QgaQzw3yQd026OiCvqbd8auA0YSTKExdiIWJlnTGZWfl26dGHUqFGMGjWKvffemylTpjB+/HgGDBjA4MGDefjhh/n1r3/N448/DsBee+3FU089xaZNm0reFWy55ZZs3ryZLbbYgvXr1/PHP/6Rp59+Gkls2rQJSVx11VX06tWrthG2xvr16+nduzdDhgzhxRdf5M033yx5V3DBBRcwc+bMButPOeUUJk2a1Mx3JrHLLrvUVvmsWbOGnXfeGYB+/fqxatWq2nLV1dX069cPSPqHbLPNNi06X325VQ2lw1ffAHwGGAaMkzSsXrEzgdcjYghwHXBlXvGYWftYunQpy5Ytq11esGABu+66a+3yuHHjuOCCC9htt93o378/ALvvvjtVVVVcfPHFRCSDGKxcuZLf/e53DY7/kY98hBUrVgBw1113cdppp/HCCy+wcuVKVq1axeDBg3n00UcZOnQoL730EkuWLAHghRde4KmnnmLEiBFsu+22nHnmmZx//vm88847QPJkzp133tngfNdddx0LFixo8NPSJABw3HHHMWXKFACmTJnC8ccfX7v+tttuIyKYPXs2PXv2rK1C+utf/8rw4cNbfM5CebYR7A8sj4gVEfEOMB04vl6Z44Ep6eu7gNFyzxezTmXDhg2cccYZDBs2jH322YfFixdzySWX1G7/3Oc+x6JFixpUC91888288sorDBkyhOHDhzN+/Pjab8qFjj766NqG1mnTpvHZz362zvYTTzyRadOmsfXWW3P77bfzxS9+kREjRnDSSSdx880307NnTwAuv/xy+vTpw7Bhwxg+fDjHHHNMpjaDUgYNGsTEiRO59dZb6d+/P4sXLwaSp6Lmzp0LwKRJk3jwwQcZOnQoDz30UG1SOeqoo9htt90YMmQIZ599NjfeeGPtcWfOnMnRRx/d6vgAVJNt25qkk4AxEXFWunwa8PGIOLegzDNpmep0+bm0zGv1jjUBmAAwcODAkS+88EKz4/m/RS9z94LVXHvyCLp1Ld0AZdZZLFmypPbZ/M5ozZo1nH766Tz44IPtHUpZHXroodxzzz3suOOODbYV+8wlzYuIqmLH6hBPDUXE5IioioiqPn36tOgYR+71IW78wkgnAbNOpm/fvpx99tl1OpR1dmvXrmXixIlFk0BL5NlYvBoYULDcP11XrEy1pC2BniSNxmZmmbX2ef+Opk+fPpxwwgltdrw87wieAIZKGixpK+AUYEa9MjOAM9LXJwF/jLzqqswqmP9bVY6WfNa5JYKIeA84F3gAWALcERGLJF0m6bi02M+BXpKWAxOBlje7m1lR3bp1Y926dU4GFaBmPoJu3bo1a7/cGovzUlVVFTUt7WZWmmcoqyyNzVDWVGOxexabdXJdu3Zt1mxVVnk6xFNDZmaWHycCM7MK50RgZlbhOlxjsaS1QPO7Fid6Aw1ntujcfM2VwddcGVpzzbtGRNEeuR0uEbSGpLmNtZp3Vr7myuBrrgx5XbOrhszMKpwTgZlZhau0RDC5vQNoB77myuBrrgy5XHNFtRGYmVlDlXZHYGZm9TgRmJlVuE6ZCCSNkbRU0nJJDUY0lbS1pF+l2+dIGtQOYbapDNc8UdJiSQsl/UHSrsWO05GUuuaCcidKCkkd/lHDLNcs6eT0s14k6ZfljrGtZfjdHihppqQn09/vo9ojzrYi6RZJr6YzOBbbLknXp+/HQkn7tfqkEdGpfoAuwHPAbsBWwFPAsHplzgF+mr4+BfhVe8ddhms+DNg2ff3lSrjmtFx34BFgNlDV3nGX4XMeCjwJ7Jgu79zecZfhmicDX05fDwNWtnfcrbzmQ4H9gGca2X4UcD8g4ABgTmvP2RnvCPYHlkfEioh4B5gOHF+vzPHAlPT1XcBoSSpjjG2t5DVHxMyI2JguziaZMa4jy/I5A3wXuBLoDGMwZ7nms4EbIuJ1gIh4tcwxtrUs1xxAzSzzPYGXyhhfm4uIR4D1TRQ5HrgtErOBHST1bc05O2Mi6AesKliuTtcVLRPJBDpvAL3KEl0+slxzoTNJvlF0ZCWvOb1lHhARvytnYDnK8jnvAewh6TFJsyWNKVt0+chyzZcAp0qqBu4Dvlqe0NpNc/+/l+T5CCqMpFOBKuCT7R1LniRtAVwLjG/nUMptS5LqoVEkd32PSNo7Iv7WnkHlbBxwa0RcI+lA4BeShkfE5vYOrKPojHcEq4EBBcv903VFy0jakuR2cl1ZostHlmtG0qeAbwHHRcQ/yxRbXkpdc3dgODBL0kqSutQZHbzBOMvnXA3MiIh3I+J54K8kiaGjynLNZwJ3AETE40A3ksHZOqtM/9+bozMmgieAoZIGS9qKpDF4Rr0yM4Az0tcnAX+MtBWmgyp5zZI+BvyMJAl09HpjKHHNEfFGRPSOiEERMYikXeS4iOjI85xm+d2+m+RuAEm9SaqKVpQxxraW5ZpfBEYDSNqTJBGsLWuU5TUDOD19eugA4I2IWNOaA3a6qqGIeE/SucADJE8c3BIRiyRdBsyNiBnAz0luH5eTNMqc0n4Rt17Ga74K2B64M20XfzEijmu3oFsp4zV3Khmv+QHgSEmLgU3AhRHRYe92M17z14GbJF1A0nA8viN/sZM0jSSZ907bPS4GugJExE9J2kGOApYDG4EvtvqcHfj9MjOzNtAZq4bMzKwZnAjMzCqcE4GZWYVzIjAzq3BOBGZmFc6JoAJI2iRpQcHPoCbKbmiD890q6fn0XPPT3p7NPcbNkoalr/+z3rY/tzbG9Dg178szkn4raYcS5Ue0ZGRLSX0l3Zu+HiXpjfS8SyRd3ILjHVczCqekE2rep3T5srTjYKukn+FJJcrMak4HvfTa781Qrujom5KulnR41vNZdk4EleEfETGi4GdlGc55YUSMACaRdGRrlog4KyIWp4v/WW/bQa0PD3j/fRlO0p/kKyXKjyB5fru5JgI3FSw/mr43VSRj5DRrGOGImBERV6SLJ5CMuFmz7aKIeKgFMX6Q3AoUGyPpRyS/T9bGnAgqkKTtlcxJMF/S05IajNqZfot9pOAb8yfS9UdKejzd905J25c43SPAkHTfiemxnpH0tXTddpJ+J+mpdP3YdP0sSVWSrgC2SeOYmm7bkP47XdLRBTHfKukkSV0kXSXpCSXjtX8pw9vyOOnAXZL2T6/xSUl/lvSRtFfrZcDYNJaxaey3SPpLWrbY6KcAJwK/r78yIv4OzAOGpHcbs9N4fyNpxzSW8/T+PBLT03XjJf1Y0kHAccBVaUy7F7wHYyTdWfDe1H4bb+5nKOmi9L18RtJkqc5IvacV/I7sn5bP+r4U1djomxHxAtBL0oeaczzLoD3G2/ZPeX9IepguSH9+Q9KjvEe6rTdJD8WazoUb0n+/Dnwrfd2FZOye3iR/2LdL1/8HcFGR890KnJS+/hwwBxgJPA1sR9LDeRHwMZI/kjcV7Nsz/XcW6fwBNTEVlKmJ8bPAlPT1ViQjMm4DTAC+na7fGpgLDC4S54aC67sTGJMu9wC2TF9/Cvh1+no88OOC/b8PnJq+3oFkXJ/t6p1jMDCvYHkUcG/6uhewEtgLWAh8Ml1/GfDD9PVLwNY156gfR+F7XbicfsYvFnxWPwFObeFnuFPB+l8AxxZ8Rjelrw8lHT+/sfel3rVXATc38Ts7iCLj8ZPcWZ3Y3v+nOttPpxtiwor6RyRVEQBI6gp8X9KhwGaSb8K7AC8X7PMEcEta9u6IWCDpkyTVEI+lXwq3IvkmXcxVkr5NMubLmSRjwfwmkm/BSPpf4BMk35SvkXQlyR+JR5txXfcD/y1pa5KqhEci4h+SjgT2Kajj7kky8Nrz9fbfRtKC9PqXAA8WlJ8iaSjJkAVdGzn/kcBxkr6RLncDBqbHqtGXhuPefELSkyTv/RUkA8XtEBEPp9unkCQmSBLEVEl3k4wjlEkkQzP8HjhW0l3A0cC/k4w6m/UzrHGYpH8HtgV2Ikniv023TUvP94ikHkraWRp7XwrjmwuclfV6CrwKfLgF+1kTnAgq0xeAPsDIiHhXyeic3QoLpP+xDyX5A3KrpGuB14EHI2JchnNcGBF31SxIGl2sUET8Na0jPwq4XNIfIuKyLBcREW9LmgV8GhhLMmkJJDM3fTUiHihxiH9ExAhJ25KMZfMV4HqSyWxmRsRnlTSsz2pkf5F8O13a1Dmo996StBEcU3sQqWcT+x9N8m37WOBbkvZuomx904FzSapZ5kbEW2m1TtbPEEndgBtJ7s5WSbqEutdTf4yaoJH3RdIuzYi9Md1I3lNrQ24jqEw9gVfTJHAY0GD+YiVzGr8SETcBN5NMnTcbOFhSTZ3/dpL2yHjOR4ETJG0raTuSap1HJX0Y2BgRt5MMjFes4fTd9M6kmF+RDLpVc3cByR/1L9fsI2mP9JxFRTJz23nA1/X+sOQ1w/qOLyj6FkkVWY0HgK/W1JkrGeG1vr+SVHM0KiLeAF5X2g4DnAY8rGROhQERMZOkCqcnSbVaofoxFXqY5P08m/eTZHM/w5o/+q+lbQn1nySqadM5hGQUzDfI9r601B5A0bl8reWcCCrTVKBK0tPA6cCzRcqMAp5KqzDGAv8dEWtJ/jBOk7SQpErho1lOGBHzSeqd/0LSZnBzRDwJ7A38Ja2iuRi4vMjuk4GFShuL6/k/kuqOhyKZyhCSxLUYmK/kEcSfUeLuN41lIckkJz8A/iu99sL9ZgLDahqLSe4cuqaxLUqX6x/378BzNX94m3AGSXXaQpKnky4jabu4Pf2cngSuj4YTzEwHLkwbZXevd+5NwL3AZ9J/ae5nmJ7vJpI/vg+QVBkWejt9n35KUgUIGd4XJQ8C3FzsnEpG33wc+Iikaklnpuu7kjx40JGHEv9A8uijZjmT9FmSarhvt3csHVn6Pu4XEd9p71g6G7cRmOUsIn4jqSPPif1BsSVwTXsH0Rn5jsDMrMK5jcDMrMI5EZiZVTgnAjOzCudEYGZW4ZwIzMwq3P8HFtsPY1RILPYAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "lg_parameters = {'solver': ['liblinear', 'saga'],\n",
        "                 'penalty': [\"l1\", \"l2\"],\n",
        "                 'C': [0.1, 1, 10, 100, 200],\n",
        "                 'random_state': [1234]}\n",
        "\n",
        "logreg_tune = LogisticRegression(max_iter=5000)\n",
        "lg_tune = GridSearchCV(logreg_tune,\n",
        "                        lg_parameters,\n",
        "                        cv = 3,\n",
        "                        scoring = 'roc_auc',\n",
        "                        verbose = 3)\n",
        "\n",
        "logreg = lg_tune.fit(x_train_bowandtfidf,y_train)\n",
        "\n",
        "#logreg = LogisticRegression(penalty=\"none\", max_iter=5000, random_state=1234)\n",
        "logreg.fit(x_train_bowandtfidf,y_train)\n",
        "ypred = logreg.predict(x_train_bowandtfidf)\n",
        "print('Accuracy Score:', (accuracy_score(y_train, ypred)))\n",
        "print('Precision Score:', (precision_score(y_train, ypred)))\n",
        "print('Recall Score:', (recall_score(y_train,ypred)))\n",
        "print('F1 Score:', (f1_score(y_train, ypred)))\n",
        "\n",
        "y_score = svm.predict_proba(x_train_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_train, y_score[:,1])))\n",
        "print(plot_roc_curve(svm, x_train_bowandtfidf, y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 147,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "N228jtZCN-OB",
        "outputId": "bf6fcdff-c0ae-4ad6-b844-3078b5ac094b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\hwhme\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metrics.RocCurveDisplay.from_predictions` or :meth:`sklearn.metrics.RocCurveDisplay.from_estimator`.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy Score: 0.9871364653243848\n",
            "Precision Score: 0.98\n",
            "Recall Score: 0.6901408450704225\n",
            "F1 Score: 0.8099173553719008\n",
            "ROC Score: 0.8102241872903115\n",
            "<sklearn.metrics._plot.roc_curve.RocCurveDisplay object at 0x000001A1386E6770>\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAua0lEQVR4nO3de3xU1bn/8c9DIAmXBIQEKzcJCkVBpBDwUkXUWm21okULWHvxaD0q6jle6NG2R4W29qK2XmpRaP2JrYIXaoutB6ooWLUooIhcaosUS1BLBOQihJDk+f2xd4ZJMsnskMyESb7v1ysvZt9mP3sS1jN7rbXXMndHRETarnYtHYCIiLQsJQIRkTZOiUBEpI1TIhARaeOUCERE2rj2LR1AYxUUFHj//v1bOgwRkYyyfPnyj9y9MNG2jEsE/fv3Z9myZS0dhohIRjGz9+rbpqohEZE2TolARKSNUyIQEWnjlAhERNo4JQIRkTYuZYnAzB4ys81mtqqe7WZm95rZOjNbaWYjUhWLiIjUL5V3BA8DZzWw/QvAwPDncmB6CmMREZF6pCwRuPtLwNYGdhkHPOKBJUA3MzssVfGIiGSqyirnh39aw6aP96Tk/VuyjaA3sDFuuSRcV4eZXW5my8xsWWlpaVqCExE5GLg73336bWb+5Z8sfic15V9GNBa7+wx3L3b34sLChE9Ii4i0Sj9d8A5zlm7k6lOP5KLj+qXkHC2ZCDYBfeOW+4TrREQEmPHSu0xf9C4XHdePGz4/KGXnaclEMA/4eth76Hhgu7t/0ILxiIgcNJ5YtpHbn/0bZw87jO+PG4qZpexcKRt0zsxmA2OBAjMrAW4FOgC4+wPAs8AXgXXAbuCSVMUiIpJJFqz+kJvmruTkgQX8/CvDyWqXuiQAKUwE7j4pyXYHJqfq/CIimaaqyrn/xXXc9dzf+Uy/bjxw8Uiy26e+4ibjhqEWEWmNtu/Zxw1PrOD5tZv5wtBP8aMvH0PnnPQU0UoEIiIt7G8f7uCK3yynZNsepp47hK+fcHhK2wRqUyIQEWlBf1ixiZvmvk1ebnvmXH48xf27pz0GJQIRkRawr7KK259dy/97ZQOj+h/C/ReNoGd+bovEokQgIpJmm3eWcfWjb/L6hq1c8tn+fOeLR9Ehq+V68ysRiIik0fL3tnLlb99gR9k+7pk4nHHDE46sk1aREoGZtQOOBXoBe4BV7r45lYGJiLQm7s4jf32P7/9xDb0P6cis/xjNUYflt3RYQJJEYGZHAP8DfA74B1AK5AKDzGw38CAwy92rUh2oiEgmqKpytnxSzuadZWzeuZfS8OfNf33M82v/zemDe/KzCcPp2rFDS4cak+yO4AcE8wT8Z/gAWIyZ9QQuAr4GzEpNeCIiB4e9FZWU7twbK9w379xL6Y6gsA9+yijduZePdpVTWeV1ju/asQPXnzGIq089knYpflK4sRpMBA09HRxWDd3d3AGJiKSLu7Nrb0WNwn3zjrL9r3fuf/3x7n11jjeDHp1z6JmXQ8/8HI4+LJ/CvBx65uXG1vXMy6UwL4fcDlktcIXRHHBjsZmd4e7PNWcwIiLNoarK2bq7vEbhvr+wjyvod+xlz77KOsdnZ7ULCvT8HIoKOjO6qHvCwr1H52zat2Bvn+bSlF5DvwZSMzi2iEgC5RVVfLSrZuEeFPA1C/ePdu2lIkH1TF5Oewrzg2/ww/p0Cwr2WoV7z7wcunbskNYne1tassbiefVtAno0fzgi0hZ9ElbPbE5Q514aFu6bd5axrd7qmWwKw4J80KF5cQX8/sK9Z14uHbMP3uqZlpTsjuBk4GJgV631BoxOSUQi0iq4O9t27wt6z+yoW+deGhbum3fuZXd53eqZDllGYZccCvNz6dejE8X9DwmqZ/JzKOyy/1t8jy7ZLfowVmuQLBEsAXa7++LaG8zsndSEJCIHs32VYfXMjr016t2rq2VKw9cf7drLvsq61TOds7Ni39SH9u5ao0qmunDvmZdDt05tq3qmJSXrNfSFBraNaf5wRKSl7C6vqLdwj6+m2bq7HK9bvtO9czY983IozMvhyJ55CQv3wryctA2tLNHpNyLSirk7H+/eV2/hvnnnXj4Kq2p27a2oc3z7dhYr0Psc0onP9DukTuHeMz+HHp1z0jKBiqSGEoFIBqqorOKjXeX1Fu6lcT/llXUf/O+UnRVrQD3qsHzGDKpbuBd2yeGQTtkH3cNP0vyUCEQOInvKK2sV7vFdJPd3ldzySeLqmUM6dYg90DSgoHPYVXJ/tUx1T5ouqp6ROPprEEkxd2fHnor9hXtcL5r4Qr90x152JqieyWoX9p7Jy6FX11yG9+1KYYLCvaBLNjnt1T1SGi9yIjCz29z9tvqWRdqayipny669DRbum3fspXTXXsor6lbP5HZoF/u2PvhTeYwZWEhhfOEedpXsruoZSbHG3BEsT7Is0iqU7Yurnqm3F81etn6ylwQPr9K1Y4dYPXswNEFOXAGfG9bF59Alp726R8pBIXIicPdnGloWOZi5OzvKKoI+7km6SO4oq1s9086gIHyI6VNdczmmd9dYgV4YV7gXdDm4BxcTSSTZEBP3AQm+8wTc/dpmj0ikESqrnC2fVD/IVE/hHj78tDdB9UxO+3ax3jIDe+bx2SMLYtUyhbGCPugemaXqGWmlkt0RLEtLFCK1VFfPVBfipfUU7ls+STz2e35u++Dp1S45jKju+149PEHe/m/y+bmqnhFJ9mRxjQlnzKyTu+9ObUjSWrk7O/dW1BhErMYY8HHDFmzfU3dwsXYGPbrsH2fm6MPyE449c7CP/S5ysIk6Z/EJBMNOdwH6mdmxBLOWXZXK4CQzVE/NV6MrZILCffPOMsr21a2eyW7fLlaQDyjszPEDeuzvGhnXD757Kxn7XeRgE7Wx+G7gTGAegLu/ZWYaa6iVK6+oCgvyuIeaEhTu9U3NFz/2+/C+3RIW7j3zcsnvqOoZkZbUmF5DG2v9Z607bqxkhF17K2qO+x4W7qVxhXvDU/Nlxx5oGvypvBoPNcUaWvNyNPa7SIaImgg2mtmJgJtZB+C/gLWpC0saq6rK2ba7vE7hXt2bJr7KJtHY79VT8xXm5XB4j86M6t+9Rp/32NR8GvtdpNWJmgiuAO4BegPvAwuAyakKSvarPTVfaY1x4GsOMJZoar4uOe2D/u15OUHf9wSFu8Z+F2nbIiUCd/8I+Gpj39zMziJIIFnAr9z9x7W29wNmAd3CfW5y92cbe55MFD81X+069/394fey9ZPyhMcH1TPBN/iBPfMSFu4983PolK3hpESkYVF7DQ0gKNCPJ3jA7K/Ade6+voFjsoD7gTOAEmCpmc1z9zVxu30PeMLdp5vZ0cCzQP8DuZCDQfzUfPu7SNYdGnjzjjI+SVA9U3vs9xGHHxI3sNj+4YELuuSoekZEmk3Ur4uPERTq54fLE4HZwHENHDMaWFedLMxsDjAOiE8EDuSHr7sSVDsddPZVVrFlV3mduVdrDA0cfrNPNDVf/NjvR/fKZ+ynC+sU7j3zcunWsYMGFxORtIuaCDq5+2/iln9rZlOSHNMb2Bi3XELdxHEb8GczuwboDHwu0RuZ2eXA5QD9+vWLGHJyu8sr4h5oqlu4b95R1uDUfId06hCrcz+isEfNwj3sRVMYDi4mInKwSjbWUPfw5f+Z2U3AHIJv8RMIqnGaahLwsLvfFT609hszG+ruNZ46cvcZwAyA4uLiesc+Smbd5l3cOm8VH3xcVu/UfNVjv/fMz6HPIR35TL9usa6S8V0kC7poaj4RaR2SfVVdTlDwV9dX/GfcNgdubuDYTUDfuOU+4bp4lwJnAbj7X80sFygANieJ64As3bCVV9ZtofjwQxgzqLBG//fqwl9jv4tIW5NsrKGiJrz3UmCgmRURJICJwEW19vkXcDrwsJkdBeQCpU04ZyS/uGgEn+qam+rTiIhkhMbMUDYUOJqgsAbA3R+pb393rzCzqwmeOcgCHnL31WY2DVjm7vOAG4CZZnYdwR3GN90T1caLiEiqRO0+eiswliARPAt8AXgZqDcRAITPBDxba90tca/XAJ9tVMQiItKsorZ2XkBQhfOhu18CHEvQ3VNERDJc1ESwJ+zJU2Fm+QSNuX2THCMiIhkgahvBMjPrBswk6Em0i+DpYhERyXBRxxqqnoDmATObD+S7+8rUhSUiIumS7IGyEQ1tc/c3mj+k1KkenbOdngMTEYlJdkdwVwPbHDitGWNJuR3hPLj5uR1aOBIRkYNHsgfKTk1XIOnw9JubaGeQo6EhRERi2lSJ+K+tu8lu304TsIiIxGkziWBfZRXlFVVcNfbIlg5FROSg0mYSwc6yYKTR/FwNCS0iEi9SIrDAxWZ2S7jcz8xGpza05rU9bCju2kkNxSIi8aLeEfwSOIFg/gCAnQQzlmUM9RgSEUksaj3Jce4+wszeBHD3bWaWncK4mt2OsiAR5CkRiIjUEPWOYF84Gb0DmFkhUNXwIQeXyvBhsvZZ6jEkIhIvaiK4F3ga6GlmPyQYgvr2lEUlIiJpE3WsoUfNbDnBUNQGnOfua1MamYiIpEXUiWnuBea4e0Y1EIuISHJRq4aWA98zs3fN7E4zK05lUCIikj6REoG7z3L3LwKjgHeAn5jZP1IamYiIpEVjnyw+EhgMHA78rfnDERGRdIv6ZPFPwzuAacAqoNjdv5TSyEREJC2iPlD2LnCCu3+UymBERCT9ks1QNtjd/wYsBfqZWb/47Zk2Q5mIiNSV7I7geuByEs9UlnEzlImISF3JZii7PHz5BXcvi99mZrkpi0pERNImaq+hVyOuExGRDJOsjeBTQG+go5l9hmB4CYB8oFOKYxMRkTRI1kZwJvBNoA/ws7j1O4HvpCgmERFJo2RtBLOAWWY23t3npikmERFJo2RVQxe7+2+B/mZ2fe3t7v6zBIeJiEgGSdZY3Dn8twuQl+CnQWZ2lpm9Y2brzOymevb5ipmtMbPVZvZYI2IXEZFmkKxq6MHw36mNfeNwRrP7gTOAEmCpmc1z9zVx+wwEbgY+G05/2bOx5xERkaZpzFhD+WbWwcwWmlmpmV2c5LDRwDp3X+/u5cAcYFytfb4F3O/u2wDcfXNjL0BERJom6nMEn3f3HcA5wAaCUUinJDmmN7AxbrkkXBdvEDDIzF4xsyVmdlaiNzKzy81smZktKy0tjRiyiIhEETURVFchnQ086e7bm+n87YGBwFhgEjDTzLrV3sndZ7h7sbsXFxYWNtOpRUQEoieCP5rZ34CRwEIzKwTKkhyzCegbt9wnXBevBJjn7vvc/Z/A3wkSg4iIpEnUGcpuAk4kmIdgH/AJdev7a1sKDDSzIjPLBiYC82rt83uCuwHMrICgqmh91OBFRKTpok5e3wG4GBhjZgCLgQcaOsbdK8zsamABkAU85O6rzWwasMzd54XbPm9ma4BKYIq7bzngqxERkUaLOjHNdKAD8Mtw+WvhussaOsjdnwWerbXulrjXTjDUdZ2H1UREJD2iJoJR7n5s3PILZvZWKgISEZH0itpYXGlmR1QvmNkAgqocERHJcFHvCKYAL5rZeoKhqA8HLklZVCIikjZJE0HYVXQ7wZPC1UNAvOPue1MZmIiIpEeDVUNmdhmwGrgPWAH0d/eVSgIiIq1HsjuC/waGuHtp2C7wKHWfBRARkQyWrLG43N1LAdx9PZCT+pBERCSdkt0R9DGze+tbdvdrUxOWiIikS7JEUHuE0eWpCkRERFpGlDmLWwVv6QBERA5SyXoNzTSzofVs62xm/2FmX01NaM0rGM0C2gVjJYmISChZ1dD9wC1mdgywCigFcgmGis4HHiLoSXTQq6wK/s1SIhARqSFZ1dAK4Ctm1gUoBg4D9gBr3f2d1IfXfCqrwjuCqINqiIi0EZGGmHD3XcCi1IaSWtWJoL0ygYhIDW2mVKwM2wiy2swVi4hE02aKxaoqNRaLiCTSqERgZp1SFUiqVVcNZbVTIhARiRcpEZjZieF0kn8Ll481s18mOeygsr9qSIlARCRe1DuCnwNnAlsA3P0tYEyqgkoF3RGIiCQWuWrI3TfWWpVRM5TFEoHaCEREaog6Q9lGMzsRcDPrAPwXsDZ1YTW/quoni3VHICJSQ9Q7giuAyUBvYBMwHLgqRTGlxP7nCJQIRETiRb0j+LS71xhTyMw+C7zS/CGlxv4ni5UIRETiRb0juC/iuoOW2ghERBJr8I7AzE4ATgQKzez6uE35QFYqA0sV5QERkZqSVQ1lA13C/fLi1u8ALkhVUCIikj7JRh9dDCw2s4fd/b00xSQiImkUtbF4t5ndAQwhmI8AAHc/LSVRiYhI2kRtLH6UYHiJImAqsAFYmqKYREQkjaImgh7u/mtgn7svdvf/AHQ3ICLSCkStGtoX/vuBmZ0NvA90T01IIiKSTlHvCH5gZl2BG4AbgV8B/53sIDM7y8zeMbN1ZnZTA/uNNzM3s+KI8YiISDOJOlXlH8OX24FTIfZkcb3MLAu4HzgDKAGWmtk8d19Ta788grGLXmtc6CIi0hwavCMwsywzm2RmN5rZ0HDdOWb2KvCLJO89Gljn7uvdvRyYA4xLsN/3gZ8AZY0PX0REmipZ1dCvgcuAHsC9ZvZb4E7gp+7+mSTH9gbih64uCdfFmNkIoK+7/6mhNzKzy81smZktKy0tTXJaERFpjGRVQ8XAMHevMrNc4EPgCHff0tQTm1k74GfAN5Pt6+4zgBkAxcXF3tRzi4jIfsnuCMrdvQrA3cuA9Y1IApuAvnHLfcJ11fKAocAiM9sAHA/MU4OxiEh6JbsjGGxmK8PXBhwRLhvg7j6sgWOXAgPNrIggAUwELqre6O7bgYLqZTNbBNzo7ssafRUiInLAkiWCow70jd29wsyuBhYQjFT6kLuvNrNpwDJ3n3eg7y0iIs0n2aBzTRpozt2fBZ6tte6WevYd25RziYjIgYk8eb2IiLROSgQiIm1c5ERgZh3N7NOpDEZERNIvUiIwsy8BK4D54fJwM1Njr4hIKxD1juA2giEjPgZw9xUEcxOIiEiGi5oI9oX9/uPpCV8RkVYg6nwEq83sIiDLzAYC1wKvpi4sERFJl6h3BNcQzFe8F3iMYDjq/05RTCIikkZR7wgGu/t3ge+mMhgREUm/qHcEd5nZWjP7fvW8BCIi0jpESgTufirBzGSlwINm9raZfS+lkYmISFpEfqDM3T9093uBKwieKUg4ZpCIiGSWqA+UHWVmt5nZ28B9BD2G+qQ0MhERSYuojcUPAY8DZ7r7+ymMR0RE0ixSInD3E1IdiIiItIwGE4GZPeHuXwmrhOKfJI4yQ5mIiGSAZHcE/xX+e06qAxERkZbRYGOxu38QvrzK3d+L/wGuSn14IiKSalG7j56RYN0XmjMQERFpGcnaCK4k+OY/wMxWxm3KA15JZWAiIpIeydoIHgP+D/gRcFPc+p3uvjVlUYmISNokSwTu7hvMbHLtDWbWXclARCTzRbkjOAdYTtB91OK2OTAgRXGJiEiaNJgI3P2c8F9NSyki0kpFHWvos2bWOXx9sZn9zMz6pTY0ERFJh6jdR6cDu83sWOAG4F3gNymLSkRE0iZqIqhwdwfGAb9w9/sJupCKiEiGizr66E4zuxn4GnCymbUDOqQuLBERSZeodwQTCCau/w93/5BgLoI7UhaViIikTdSpKj8EHgW6mtk5QJm7P5LSyEREJC2i9hr6CvA6cCHwFeA1M7sgwnFnmdk7ZrbOzG5KsP16M1tjZivNbKGZHd7YCxARkaaJ2kbwXWCUu28GMLNC4HngqfoOMLMs4H6CAetKgKVmNs/d18Tt9iZQ7O67w3GNfkpQDSUiImkStY2gXXUSCG2JcOxoYJ27r3f3cmAOQa+jGHd/0d13h4tL0DzIIiJpF/WOYL6ZLQBmh8sTgGeTHNMb2Bi3XAIc18D+lxIMcFeHmV0OXA7Qr5+eYxMRaU5R5yyeYmZfBk4KV81w96ebKwgzuxgoBk6p5/wzgBkAxcXFnmgfERE5MMnmIxgI3AkcAbwN3OjumyK+9yagb9xyn3Bd7XN8jqAN4hR33xvxvUVEpJkkq+d/CPgjMJ5gBNL7GvHeS4GBZlZkZtnARGBe/A5m9hngQeDcWm0QIiKSJsmqhvLcfWb4+h0zeyPqG7t7hZldDSwAsoCH3H21mU0Dlrn7PIKH0roAT5oZwL/c/dxGX4WIiBywZIkgN/zWXj0PQcf4ZXdvMDG4+7PUalR291viXn+u0RGLiEizSpYIPgB+Frf8YdyyA6elIigREUmfZBPTnJquQEREpGVEfaBMRERaKSUCEZE2TolARKSNizr6qIVzFd8SLvczs9GpDU1ERNIh6h3BL4ETgEnh8k6CkUVFRCTDRR107jh3H2FmbwK4+7bwaWEREclwUe8I9oXzCzjE5iOoSllUIiKSNlETwb3A00BPM/sh8DJwe8qiEhGRtIk6DPWjZrYcOJ1geInz3H1tSiMTEZG0iJQIzKwfsBt4Jn6du/8rVYGJiEh6RG0s/hNB+4ABuUAR8A4wJEVxiYhImkStGjomftnMRgBXpSQiERFJqwN6sjgcfrqh+YdFRCRDRG0juD5usR0wAng/JRGJiEhaRW0jyIt7XUHQZjC3+cMREZF0S5oIwgfJ8tz9xjTEIyIiadZgG4GZtXf3SuCzaYpHRETSLNkdwesE7QErzGwe8CTwSfVGd/9dCmMTEZE0iNpGkAtsIZijuPp5AgeUCEREMlyyRNAz7DG0iv0JoJqnLCqRg8S+ffsoKSmhrKyspUMRiSQ3N5c+ffrQoUOHyMckSwRZQBdqJoBqSgTS6pWUlJCXl0f//v0xS/TfQOTg4e5s2bKFkpISioqKIh+XLBF84O7TmhaaSOYqKytTEpCMYWb06NGD0tLSRh2X7Mli/fVLm6ckIJnkQP5ekyWC0w8sFBERyRQNJgJ335quQEQksX//+99cdNFFDBgwgJEjR3LCCSfw9NNPJ9z3/fff54ILLki4bezYsSxbtgyAhx56iGOOOYZhw4YxdOhQ/vCHP6Qs/g0bNjB06NB6t995550MHjyY4cOHM2rUKB555BGmTp3KzTffXGO/FStWcNRRRyV8jwsuuID169fX2NfMmD9/foNx3Hbbbdx5550NxtJUs2bNYuDAgQwcOJBZs2Yl3GfFihUcf/zxDB8+nOLiYl5//fXYtkWLFjF8+HCGDBnCKaecAkB5eTljxoyhoqKiyfHBAQ46JyLp4e6cd955jBkzhvXr17N8+XLmzJlDSUlJnX0rKiro1asXTz31VIPvWVJSwg9/+ENefvllVq5cyZIlSxg2bFiTYz2QQumBBx7gueee4/XXX2fFihUsXLgQd2fSpEk8/vjjNfadM2cOkyZNqvMeq1evprKykgEDBsTWzZ49m5NOOonZs2c3OZam2Lp1K1OnTuW1117j9ddfZ+rUqWzbtq3Oft/+9re59dZbWbFiBdOmTePb3/42AB9//DFXXXUV8+bNY/Xq1Tz55JMAZGdnc/rpp9f5jA5U1OcIRNq8qc+sZs37O5r1PY/ulc+tX6p/Wo8XXniB7Oxsrrjiiti6ww8/nGuuuQaAhx9+mN/97nfs2rWLyspKZs2axTnnnMOqVavYs2cPl1xyCW+99RaDBw9mz549AGzevJm8vDy6dOkCQJcuXWKv3333XSZPnkxpaSmdOnVi5syZDB48mGeeeYYf/OAHlJeX06NHDx599FEOPfRQbrvtNt59913Wr19Pv379uPvuu7niiiti386nT59Or169qKys5Fvf+havvvoqvXv35g9/+AMdO3bk9ttvZ9GiReTn5wOQn5/PN77xDQAOOeQQXnvtNY47Lhjo+IknnmDBggV1PqNHH32UcePGxZbdnSeffJLnnnuOk08+mbKyMnJzc5P+LhqK5UAtWLCAM844g+7duwNwxhlnMH/+/DoJzczYsSP429q+fTu9evUC4LHHHuPLX/4y/fr1A6Bnz56xY8477zxuvvlmvvrVrzYpRtAdgchBbfXq1YwYMaLBfd544w2eeuopFi9eXGP99OnT6dSpE2vXrmXq1KksX74cgGOPPZZDDz2UoqIiLrnkEp55JjbxIJdffjn33Xcfy5cv58477+Sqq4JpR0466SSWLFnCm2++ycSJE/npT38aO2bNmjU8//zzzJ49m2uvvZZTTjmFt956izfeeIMhQ4Ik949//IPJkyezevVqunXrxty5c9mxYwc7d+6s8U0+3qRJk5gzZw4AS5YsoXv37gwcOLDOfq+88gojR46MLb/66qsUFRVxxBFHMHbsWP70pz81+PkBSWOJd8cddzB8+PA6P9dee22dfTdt2kTfvn1jy3369GHTpk119rv77ruZMmUKffv25cYbb+RHP/oRAH//+9/Ztm0bY8eOZeTIkTWqqoYOHcrSpUuTxhuF7ghEImrom3u6TJ48mZdffpns7OxYIRD/jTPeSy+9FCuchg0bFqv+ycrKYv78+SxdupSFCxdy3XXXsXz5cm688UZeffVVLrzwwth77N27FwiqkyZMmMAHH3xAeXl5jT7q5557Lh07dgSCO5jqwiorK4uuXbuybds2ioqKGD58OAAjR45kw4YNSa91woQJnHjiidx11131VgsBfPDBBxQWFsaWZ8+ezcSJEwGYOHEijzzyCOPHj6+3N01je9lMmTKFKVOmNOqYZKZPn87Pf/5zxo8fzxNPPMGll17K888/T0VFBcuXL2fhwoXs2bOHE044geOPP55BgwaRlZVFdnY2O3fuJC8vL/lJGpDSOwIzO8vM3jGzdWZ2U4LtOWb2eLj9NTPrn8p4RDLNkCFDeOONN2LL999/PwsXLqzRT7xz586Nfl8zY/To0dx8883MmTOHuXPnUlVVRbdu3VixYkXsZ+3atQBcc801XH311bz99ts8+OCDNZ60jnL+nJyc2OusrCwqKirIz8+nS5cuNRp54/Xt25eioiIWL17M3LlzmTBhQsL9OnbsGIunsrKSuXPnMm3aNPr3788111zD/Pnz2blzJz169KhTP79161YKCgqSxhKvMXcEvXv3ZuPGjbHlkpISevfuXWe/WbNm8eUvfxmACy+8MNZY3KdPH84880w6d+5MQUEBY8aM4a233oodt3fv3kjVXsmkLBGEw1ffD3wBOBqYZGZH19rtUmCbux8J/Bz4SariEclEp512GmVlZUyfPj22bvfu3ZGOHTNmDI899hgAq1atYuXKlUDQsyg+uaxYsYLDDz+c/Px8ioqKYg2S7h4rdLZv3x4rwOrr+QJw+umnx2KtrKxk+/btDcZ48803M3ny5Fj9+K5du2pUf0yaNInrrruOAQMG0KdPn4TvcdRRR7Fu3ToAFi5cyLBhw9i4cSMbNmzgvffeY/z48Tz99NN06dKFww47jBdeeAEIksD8+fM56aSTIsVSbcqUKTWSZfXPvffeW2ffM888kz//+c9s27aNbdu28ec//5kzzzyzzn69evWKVe298MILsSqwcePG8fLLL1NRUcHu3bt57bXXYj2ntmzZQkFBQaOGkqhPKu8IRgPr3H29u5cDc4BxtfYZB1T/VT0FnG56ekckxsz4/e9/z+LFiykqKmL06NF84xvf4Cc/Sf6d6corr2TXrl0cddRR3HLLLbF69H379nHjjTfGukk+/vjj3HPPPUDQ8PrrX/+aY489liFDhsS6ld52221ceOGFjBw5koKCgnrPec899/Diiy9yzDHHMHLkSNasWZM0xlNPPZVRo0YxdOhQTj75ZNq1218sXXjhhaxevbreaiGAs88+m0WLFgFBtdD5559fY/v48eNjvYceeeQRvv/97zN8+HBOO+00br31Vo444ohIsRyI7t2787//+7+MGjWKUaNGccstt8Sq8S677LJYd96ZM2dyww03cOyxx/Kd73yHGTNmAEGSO+ussxg2bBijR4/msssui3WBffHFFzn77LObFF+Mu6fkB7gA+FXc8teAX9TaZxXQJ275XaAgwXtdDiwDlvXr188PxIJVH/iVv13me8orDuh4aZvWrFnT0iFIErt37/bjjjvOKyra1v/t888/3995552E2xL93QLLvJ7yOiN6Dbn7DHcvdvfi+Eahxvj8kE/xy6+OJLdDVjNHJyItqWPHjkydOjVhb5zWqry8nPPOO49BgwY1y/ulstfQJqBv3HKfcF2ifUrMrD3QlWDeAxGRyBLVu7dm2dnZfP3rX2+290vlHcFSYKCZFZlZNjARmFdrn3lA9RMbFwAvhLcwIgcN/UlKJjmQv9eUJQJ3rwCuBhYAa4En3H21mU0zs3PD3X4N9DCzdcD1QJ0upiItKTc3ly1btigZSEbwcD6CxnYptUz7Ay8uLvbqlnaRVNMMZZJp6puhzMyWu3txomP0ZLFIAzp06NComZ5EMlFG9BoSEZHUUSIQEWnjlAhERNq4jGssNrNS4L0DPLwA+KgZw8kEuua2QdfcNjTlmg9394RP5GZcImgKM1tWX6t5a6Vrbht0zW1Dqq5ZVUMiIm2cEoGISBvX1hLBjJYOoAXomtsGXXPbkJJrblNtBCIiUldbuyMQEZFalAhERNq4VpkIzOwsM3vHzNaZWZ0RTc0sx8weD7e/Zmb9WyDMZhXhmq83szVmttLMFprZ4S0RZ3NKds1x+403MzezjO9qGOWazewr4e96tZk9lu4Ym1uEv+1+Zvaimb0Z/n1/sSXibC5m9pCZbTazVfVsNzO7N/w8VprZiCaftL6pyzL1B8gimPJyAJANvAUcXWufq4AHwtcTgcdbOu40XPOpQKfw9ZVt4ZrD/fKAl4AlQHFLx52G3/NA4E3gkHC5Z0vHnYZrngFcGb4+GtjQ0nE38ZrHACOAVfVs/yLwf4ABxwOvNfWcrfGOYDSwzt3Xu3s5MAcYV2ufccCs8PVTwOlmZmmMsbklvWZ3f9Hdd4eLSwhmjMtkUX7PAN8HfgK0hnGko1zzt4D73X0bgLtvTnOMzS3KNTuQH77uCryfxvianbu/BGxtYJdxwCMeWAJ0M7PDmnLO1pgIegMb45ZLwnUJ9/FgAp3tQI+0RJcaUa453qUE3ygyWdJrDm+Z+7r7n9IZWApF+T0PAgaZ2StmtsTMzkpbdKkR5ZpvAy42sxLgWeCa9ITWYhr7/z0pzUfQxpjZxUAxcEpLx5JKZtYO+BnwzRYOJd3aE1QPjSW463vJzI5x949bMqgUmwQ87O53mdkJwG/MbKi7V7V0YJmiNd4RbAL6xi33Cdcl3MfM2hPcTm5JS3SpEeWaMbPPAd8FznX3vWmKLVWSXXMeMBRYZGYbCOpS52V4g3GU33MJMM/d97n7P4G/EySGTBXlmi8FngBw978CuQSDs7VWkf6/N0ZrTARLgYFmVmRm2QSNwfNq7TMP+Eb4+gLgBQ9bYTJU0ms2s88ADxIkgUyvN4Yk1+zu2929wN37u3t/gnaRc909k+c5jfK3/XuCuwHMrICgqmh9GmNsblGu+V/A6QBmdhRBIihNa5TpNQ/4eth76Hhgu7t/0JQ3bHVVQ+5eYWZXAwsIehw85O6rzWwasMzd5wG/Jrh9XEfQKDOx5SJuuojXfAfQBXgybBf/l7uf22JBN1HEa25VIl7zAuDzZrYGqASmuHvG3u1GvOYbgJlmdh1Bw/E3M/mLnZnNJkjmBWG7x61ABwB3f4CgHeSLwDpgN3BJk8+ZwZ+XiIg0g9ZYNSQiIo2gRCAi0sYpEYiItHFKBCIibZwSgYhIG6dE0AaYWaWZrYj76d/Avrua4XwPm9k/w3O9ET7t2dj3+JWZHR2+/k6tba82Ncbwfao/l1Vm9oyZdUuy//ADGdnSzA4zsz+Gr8ea2fbwvGvN7NYDeL9zq0fhNLPzqj+ncHla+OBgk4S/wwuS7LOoMQ/ohdf+xwj7JRx908zuNLPTop5PolMiaBv2uPvwuJ8NaTjnFHcfDtxE8CBbo7j7Ze6+Jlz8Tq1tJzY9PGD/5zKU4HmSyUn2H07Qf7uxrgdmxi3/JfxsignGyGnUMMLuPs/dfxwunkcw4mb1tlvc/fkDiPFg8jCQaIyk+wj+nqSZKRG0QWbWxYI5Cd4ws7fNrM6oneG32JfivjGfHK7/vJn9NTz2STPrkuR0LwFHhsdeH77XKjP773BdZzP7k5m9Fa6fEK5fZGbFZvZjoGMYx6Phtl3hv3PM7Oy4mB82swvMLMvM7jCzpRaM1/6fET6WvxIO3GVmo8NrfNPMXjWzT4dPtU4DJoSxTAhjf8jMXg/3TTT6KcB4YH7tle7+CbAcODK821gSxvu0mR0SxnKt7Z9HYk647ptm9gszOxE4F7gjjOmIuM/gLDN7Mu6ziX0bb+zv0MxuCT/LVWY2w6zGSL1fi/sbGR3uH/VzSai+0Tfd/T2gh5l9qjHvJxG0xHjb+knvD8ETpivCn6cJnijPD7cVEDyhWP1w4a7w3xuA74avswjG7ikgKNg7h+v/B7glwfkeBi4IX18IvAaMBN4GOhM84bwa+AxBITkz7tiu4b+LCOcPqI4pbp/qGM8HZoWvswlGZOwIXA58L1yfAywDihLEuSvu+p4EzgqX84H24evPAXPD198EfhF3/O3AxeHrbgTj+nSudY4iYHnc8ljgj+HrHsAGYAiwEjglXD8NuDt8/T6QU32O2nHEf9bxy+Hv+F9xv6vpwMUH+DvsHrf+N8CX4n5HM8PXYwjHz6/vc6l17cXArxr4m+1PgvH4Ce6sxrf0/6nW9tPqhpiQhPZ4UBUBgJl1AG43szFAFcE34UOBD+OOWQo8FO77e3dfYWanEFRDvBJ+Kcwm+CadyB1m9j2CMV8uJRgL5mkPvgVjZr8DTib4pnyXmf2EoJD4SyOu6/+Ae8wsh6Aq4SV332NmnweGxdVxdyUYeO2ftY7vaGYrwutfCzwXt/8sMxtIMGRBh3rO/3ngXDO7MVzOBfqF71XtMOqOe3Oymb1J8Nn/mGCguG7uvjjcPosgMUGQIB41s98TjCMUiQdDM8wHvmRmTwFnA98mGHU26u+w2qlm9m2gE9CdIIk/E26bHZ7vJTPLt6Cdpb7PJT6+ZcBlUa8nzmag1wEcJw1QImibvgoUAiPdfZ8Fo3Pmxu8Q/sceQ1CAPGxmPwO2Ac+5+6QI55ji7k9VL5jZ6Yl2cve/h3XkXwR+YGYL3X1alItw9zIzWwScCUwgmLQEgpmbrnH3BUneYo+7DzezTgRj2UwG7iWYzOZFdz/fgob1RfUcbwTfTt9p6BzU+mwJ2gjOib2JWdcGjj+b4Nv2l4DvmtkxDexb2xzgaoJqlmXuvjOs1on6O8TMcoFfEtydbTSz26h5PbXHqHHq+VzM7NBGxF6fXILPVJqR2gjapq7A5jAJnArUmb/YgjmN/+3uM4FfEUydtwT4rJlV1/l3NrNBEc/5F+A8M+tkZp0JqnX+Yma9gN3u/luCgfESNZzuC+9MEnmcYNCt6rsLCAr1K6uPMbNB4TkT8mDmtmuBG2z/sOTVw/p+M27XnQRVZNUWANdU15lbMMJrbX8nqOaol7tvB7ZZ2A4DfA1YbMGcCn3d/UWCKpyuBNVq8WrHFG8xwef5LfYnycb+DqsL/Y/CtoTaPYmq23ROIhgFczvRPpcDNQhIOJevHDglgrbpUaDYzN4Gvg78LcE+Y4G3wiqMCcA97l5KUDDONrOVBFUKg6Oc0N3fIKh3fp2gzeBX7v4mcAzwelhFcyvwgwSHzwBWWthYXMufCao7nvdgKkMIEtca4A0LuiA+SJK73zCWlQSTnPwU+FF47fHHvQgcXd1YTHDn0CGMbXW4XPt9PwHerS54G/ANguq0lQS9k6YRtF38Nvw9vQnc63UnmJkDTAkbZY+ode5K4I/AF8J/aezvMDzfTILCdwFBlWG8svBzeoCgChAifC4WdAT4VaJzWjD65l+BT5tZiZldGq7vQNDxIJOHEj8oafRRkRQzs/MJquG+19KxZLLwcxzh7v/b0rG0NmojEEkxd3/azDJ5TuyDRXvgrpYOojXSHYGISBunNgIRkTZOiUBEpI1TIhARaeOUCERE2jglAhGRNu7/A3Fp4/MjkK8iAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "ypred_test = logreg.predict(X_test_bowandtfidf)\n",
        "\n",
        "print('Accuracy Score:', (accuracy_score(y_test, ypred_test)))\n",
        "print('Precision Score:', (precision_score(y_test, ypred_test)))\n",
        "print('Recall Score:', (recall_score(y_test,ypred_test)))\n",
        "print('F1 Score:', (f1_score(y_test, ypred_test)))\n",
        "\n",
        "y_score = svm.predict_proba(X_test_bowandtfidf)\n",
        "print('ROC Score:', (roc_auc_score(y_test, y_score[:,1])))\n",
        "print(plot_roc_curve(logreg, X_test_bowandtfidf, y_test))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 148,
      "metadata": {
        "id": "wZICZnZvP__E"
      },
      "outputs": [],
      "source": [
        "# Accuracy Score: 0.9860178970917226\n",
        "# Precision Score: 0.927536231884058\n",
        "# Recall Score: 0.7619047619047619\n",
        "# F1 Score: 0.8366013071895425\n",
        "# ROC Score: 0.9262659289067741"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 149,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hnskR8gmN-S_",
        "outputId": "3ad3d8ed-6610-4274-8491-94af8ad9b609"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'C': 100, 'penalty': 'l1', 'random_state': 1234, 'solver': 'saga'}\n"
          ]
        }
      ],
      "source": [
        "print(logreg.best_params_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMYPmp7F4R4D"
      },
      "source": [
        "#Conclusion\n",
        "The first level of the Ensemble comprises of 5 BOW matrices fed into the 5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "itoXkeS4Fc_f"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Spc_FxoyFfSO"
      },
      "source": [
        "# Experiment (old codes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 150,
      "metadata": {
        "id": "pACCGqlKF1LJ"
      },
      "outputs": [],
      "source": [
        "#this section was the old code which was used in an attempt to perform the first version of NLP. \n",
        "#import re\n",
        "#import string\n",
        "#import numpy as np\n",
        "#import pandas as pd\n",
        "#import random\n",
        "#import matplotlib.pyplot as plt\n",
        "#import seaborn as sns\n",
        "#from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "#from sklearn.model_selection import train_test_split\n",
        "#from sklearn.pipeline import Pipeline\n",
        "#from sklearn.base import TransformerMixin\n",
        "#from sklearn.metrics import accuracy_score, plot_confusion_matrix, #classification_report, confusion_matrix\n",
        "#import spacy\n",
        "#from spacy.lang.en.stop_words import STOP_WORDS\n",
        "#from spacy.lang.en import English\n",
        "\n",
        "#import nltk\n",
        "#nltk.download('omw-1.4')\n",
        "#nltk.download('wordnet')\n",
        "#from nltk.stem import WordNetLemmatizer \n",
        "#from nltk.stem.snowball import SnowballStemmer\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 151,
      "metadata": {
        "id": "dp2Ovmn0GAvZ"
      },
      "outputs": [],
      "source": [
        "#from google.colab import drive\n",
        "#drive.mount('/content/drive')\n",
        "#path = \"/content/drive/MyDrive/mystuff/bt4012/\"\n",
        "#df=pd.read_csv(path + 'fake_job_postings.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 152,
      "metadata": {
        "id": "3WX8HBaHGBeQ"
      },
      "outputs": [],
      "source": [
        "#df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 153,
      "metadata": {
        "id": "oQSvxffNGLGu"
      },
      "outputs": [],
      "source": [
        "#df.isnull().sum()\n",
        "#Clean up dataset for NLP\n",
        "#df['text']=df['title']+' '+df['company_profile']+' '+df['description']+' '+df['requirements']+' '+df['benefits']\n",
        "#df_nlp0 = df[['text','job_id','fraudulent',]].copy()\n",
        "#verify clean up \n",
        "#print(df_nlp0.head())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 154,
      "metadata": {
        "id": "vCXZpV7aGTml"
      },
      "outputs": [],
      "source": [
        "#import spacy\n",
        "#from spacy.lang.en.stop_words import STOP_WORDS\n",
        "#from spacy.lang.en import English\n",
        "#stop_words = spacy.lang.en.stop_words.STOP_WORDS\n",
        "#punctuation = string.punctuation\n",
        "\n",
        "#def spacy_tokenizer(sentence):\n",
        "#    tokens = English(sentence)\n",
        "#    tokens = [ word.lemma_.lower().strip() \n",
        "#              if word.lemma_ != \"-PRON-\" \n",
        "#              else word.lower_ for word in tokens]\n",
        "#    tokens = [ word for word in tokens \n",
        "#              if word not in stop_words and word not in punctuation]\n",
        "#    return tokens\n",
        "    \n",
        "#def clean_text(text):\n",
        "#    return text.strip().lower()\n",
        "\n",
        "#df_nlp0['text'] = df_nlp0['text'].apply(clean_text)\n",
        "#df_nlp0.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 155,
      "metadata": {
        "id": "vdQXYMUdGdhX"
      },
      "outputs": [],
      "source": [
        "#from sklearn.model_selection import train_test_split\n",
        "#from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "#tf = TfidfVectorizer(max_features = 150)\n",
        "#t = tf.fit_transform(df_nlp0['text'])\n",
        "#df_nlp2 = pd.DataFrame(t.toarray(), columns=tf.get_feature_names_out())\n",
        "#df_nlp0.drop(['text'], axis=1, inplace=True)\n",
        "#df_nlp_t = pd.concat([df_nlp2,df_nlp0], axis=1)\n",
        "#df_nlp_t.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 156,
      "metadata": {
        "id": "aAdgMhpjGsXy"
      },
      "outputs": [],
      "source": [
        "#Y = df_nlp_t.iloc[:,-1]\n",
        "#X = df_nlp_t.iloc[:,:-1]\n",
        "\n",
        "#X_train, X_test, y_train, y_test = train_test_split(X,Y, test_size=0.4, random_state=1)\n",
        "#X_test, X_val, y_test, y_val = train_test_split(X_test,Y_test,test_size=0.5, random_state=42) \n",
        "\n",
        "#print(\"X_train\",X_train.shape)\n",
        "#print(\"y_Train:\",y_train.shape)\n",
        "#print(\"X_Test\",X_test.shape)\n",
        "#print(\"y_test:\",y_test.shape)\n",
        "# print(\"X_val\",X_val.shape)\n",
        "# print(\"y_val\", y_val.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 157,
      "metadata": {
        "id": "hW-5QDURGzZ_"
      },
      "outputs": [],
      "source": [
        "#Log regress\n",
        "#from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "#LR = LogisticRegression(C=5e1, solver='lbfgs',multi_class='multinomial', random_state=17, n_jobs=4)\n",
        "#LR_model=LR.fit(X_train, y_train)\n",
        "#LR_pred_test = LR_model.predict(X_test)\n",
        "# LR_pred_val = LR_model.predict(X_val)\n",
        "#LR_score_test = accuracy_score(y_test, LR_pred_test)\n",
        "# LR_score_val = accuracy_score(y_val, LR_pred_val)\n",
        "#LR_cr_report_test = classification_report(y_test,LR_pred_test)\n",
        "# LR_cr_report_val = classification_report(y_val,LR_pred_val)\n",
        "#LR_cf_matrix_test = confusion_matrix(y_test, LR_pred_test)\n",
        "# LR_cf_matrix_val = confusion_matrix(y_val, LR_pred_val)\n",
        "#print(\"LR Score - Test set:\",LR_score_test)\n",
        "#print(\"LR Score - Validation set:\",LR_score_val)\n",
        "#print(\"Classification Report - Test set\\n\")\n",
        "#print(LR_cr_report_test)\n",
        "# print(\"Classification Report - Validation set\\n\")\n",
        "# print(LR_cr_report_val)\n",
        "#print(\"Confusion Matrix - test set\\n\")\n",
        "#print(LR_cf_matrix_test)\n",
        "# print(\"Confusion Matrix - Validation set\\n\")\n",
        "# print(LR_cf_matrix_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 158,
      "metadata": {
        "id": "K-NphAHeG5FU"
      },
      "outputs": [],
      "source": [
        "#RFC \n",
        "#from sklearn.ensemble import RandomForestClassifier\n",
        "#rfc= RandomForestClassifier(n_jobs=3,oob_score=True, criterion=\"gini\")\n",
        "#rfc_model=rfc.fit(X_train,y_train)\n",
        "#rfc_pred_test = rfc_model.predict(X_test)\n",
        "#rfc_pred_val = rfc_model.predict(X_val)\n",
        "#rfc_score_test = accuracy_score(y_test, rfc_pred_test)\n",
        "#rfc_score_val = accuracy_score(y_val,rfc_pred_val)\n",
        "#rfc_cr_report_test=classification_report(y_test,rfc_pred_test)\n",
        "#rfc_cr_report_val=classification_report(y_val,rfc_pred_val)\n",
        "#rfc_cf_matrix_test = confusion_matrix(y_test, rfc_pred_test)\n",
        "#rfc_cf_matrix_val = confusion_matrix(y_val,rfc_pred_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 159,
      "metadata": {
        "id": "JDr4AGktG91Y"
      },
      "outputs": [],
      "source": [
        "#print(\"Score - Test set:\",rfc_score_test)\n",
        "#print(\"Score - Validation set:\",rfc_score_val)\n",
        "#print(\"Classification Report - Test set\\n\")\n",
        "#print(rfc_cr_report_test)\n",
        "# print(\"Classification Report - Validation set\\n\")\n",
        "# print(rfc_cr_report_val)\n",
        "#print(\"Confusion Matrix - test set\\n\")\n",
        "#print(rfc_cf_matrix_test)\n",
        "# print(\"Confusion Matrix - Validation set\\n\")\n",
        "# print(rfc_cf_matrix_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 160,
      "metadata": {
        "id": "HjDxUuYhHCxN"
      },
      "outputs": [],
      "source": [
        " #group_names = [\"True Neg\",\"False Pos\", \"False Neg\", \"True Pos\"]\n",
        "#group_counts_test = [\"{0:0.0f}\".format(value) for value in rfc_cf_matrix_test.flatten()]\n",
        "#group_counts_val = [\"{0:0.0f}\".format(value) for value in rfc_cf_matrix_val.flatten()]\n",
        "#group_percentages_test = [\"{0:.2%}\".format(value) for value in\n",
        "                      #rfc_cf_matrix_test.flatten()/np.sum(rfc_cf_matrix_test)]\n",
        "#group_percentages_val = [\"{0:.2%}\".format(value) for value in\n",
        "                      #rfc_cf_matrix_val.flatten()/np.sum(rfc_cf_matrix_val)]\n",
        "#labels_test = [f'{v1}\\n{v2}\\n{v3}' for v1, v2, v3 in zip(group_names,group_counts_test,group_percentages_test)]\n",
        "#labels_val = [f'{v1}\\n{v2}\\n{v3}' for v1, v2, v3 in\n",
        "          #zip(group_names,group_counts_val,group_percentages_val)]\n",
        "#labels_test = np.asarray(labels_test).reshape(2,2)\n",
        "#labels_val = np.asarray(labels_val).reshape(2,2)\n",
        "#plt.figure(figsize=(15,5))\n",
        "#plt.subplot(1,2,1)\n",
        "#plt.title(\"Confusion Matrix - Test set\")\n",
        "#sns.heatmap(rfc_cf_matrix_test, annot=labels_test,fmt='', cmap='Greens')\n",
        "#plt.subplot(1,2,2)\n",
        "# plt.title(\"Confusion Matrix - Validation Set\")\n",
        "# sns.heatmap(rfc_cf_matrix_val, annot=labels_val,fmt='', cmap='coolwarm_r')\n",
        "# plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 161,
      "metadata": {
        "id": "HJ_Bn1r5HH9H"
      },
      "outputs": [],
      "source": [
        "#Hype parameter tuning RFC\n",
        "#RFC_acc = 0\n",
        "#for criterion in ['gini', 'entropy']:\n",
        "#  for n_jobs in [ 1, 2 , 3 , 4]:\n",
        "#    for oob_score in ['True' , 'False']:\n",
        "#       RFC_model_hp = RandomForestClassifier(criterion=criterion, n_jobs=n_jobs,oob_score=oob_score)\n",
        "#       RFC_model_hp.fit(X_train, y_train)\n",
        "#       RFC_test_predictions = RFC_model_hp.predict(X_test)\n",
        "#       test_rfc = accuracy_score(y_test, RFC_test_predictions)\n",
        "#       test_rfc_cr=classification_report(y_test,RFC_test_predictions)\n",
        "#       if test_rfc > RFC_acc:\n",
        "#         RFC_parameters = f\"n_jobs:{n_jobs}, criterion:{criterion}, oob_score:{oob_score}\"\n",
        "#         RFC_acc = test_rfc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 162,
      "metadata": {
        "id": "vmLQkmb5HPE1"
      },
      "outputs": [],
      "source": [
        "#print(RFC_parameters)\n",
        "#print(\"HyperParameter tuning score (test set):\",RFC_acc ,\"\\nExisting Model score (test set) :\", rfc_score_test )\n",
        "#print(\"Classification report\\n \", test_rfc_cr )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 163,
      "metadata": {
        "id": "KfXg9gZDHRbb"
      },
      "outputs": [],
      "source": [
        "#Decision Tree \n",
        "#DecisionTreeClassifier\n",
        "#from sklearn.tree import DecisionTreeClassifier\n",
        "#DT_class = DecisionTreeClassifier(criterion='entropy',random_state=0,max_depth=10)\n",
        "#DT_class = DT_class.fit(X_train,y_train)\n",
        "#DT_pred_test = DT_class.predict(X_test)\n",
        "#DT_pred_val = DT_class.predict(X_val)\n",
        "#DT_score_test = accuracy_score(y_test, DT_pred_test)\n",
        "#DT_score_val = accuracy_score(y_val,DT_pred_val)\n",
        "#DT_cr_report_test=classification_report(y_test,DT_pred_test)\n",
        "#DT_cr_report_val=classification_report(y_val,DT_pred_val)\n",
        "#DT_cf_matrix_test = confusion_matrix(y_test, DT_pred_test)\n",
        "#DT_cf_matrix_val = confusion_matrix(y_val,DT_pred_val)\n",
        "#print(\"DT Score - Test set:\",DT_score_test)\n",
        "#print(\"DT Score - Validation set:\",DT_score_val)\n",
        "#print(\"Classification Report - Test set\\n\")\n",
        "#print(DT_cr_report_test)\n",
        "# print(\"Classification Report - Validation set\\n\")\n",
        "# print(DT_cr_report_val)\n",
        "#print(\"Confusion Matrix - test set\\n\")\n",
        "#print(DT_cf_matrix_test)\n",
        "# print(\"Confusion Matrix - Validation set\\n\")\n",
        "# print(DT_cf_matrix_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 164,
      "metadata": {
        "id": "mZH7SEnOHXHq"
      },
      "outputs": [],
      "source": [
        "#DTC_acc = 0 \n",
        "\n",
        "#for criterion in ['gini', 'entropy']:\n",
        "#  for max_depth in [7,8,9,10]:\n",
        "#    for min_samples_leaf in [5,10,20,30]:\n",
        "#      DTC_model_hp = DecisionTreeClassifier(max_depth=max_depth, criterion=criterion, min_samples_leaf=min_samples_leaf)\n",
        "#      DTC_model_hp.fit(X_train, y_train)\n",
        "#      DTC_test_predictions = DTC_model_hp.predict(X_test)\n",
        "#      test_dtc = accuracy_score(y_test, DTC_test_predictions)\n",
        "#      test_dtc_cr=classification_report(y_test,DTC_test_predictions)\n",
        "#      if test_dtc > DTC_acc:\n",
        "#        DTC_parameters = f\"criterion:{criterion}, max_depth:{max_depth},min_samples_leaf:{min_samples_leaf}\"\n",
        "#        DTC_acc = test_dtc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 165,
      "metadata": {
        "id": "SPaIrQ_rHdjI"
      },
      "outputs": [],
      "source": [
        "#print(DTC_parameters)\n",
        "#print(\"HyperParameter tuning score (test set):\",DTC_acc ,\"\\nExisting Model score (test set) :\", DT_score_test )\n",
        "#print(\"Classification report\\n \", test_dtc_cr )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 166,
      "metadata": {
        "id": "8EacqrQ0Hgoi"
      },
      "outputs": [],
      "source": [
        "#from sklearn.naive_bayes import MultinomialNB\n",
        "#nb_clf = MultinomialNB()\n",
        "#nb_clf.fit(X_train,y_train)\n",
        "#y_trainpred = nb_clf.predict(X_train) #evaluate model on training\n",
        "#y_testpred = nb_clf.predict(X_test) #evaluate on test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 167,
      "metadata": {
        "id": "ROHe0RIlHmYQ"
      },
      "outputs": [],
      "source": [
        "#nb_pred = nb_clf.predict(X_test)\n",
        "#nb_score = accuracy_score(y_test, nb_pred)\n",
        "#print(nb_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 168,
      "metadata": {
        "id": "HPxWLdAeHqMY"
      },
      "outputs": [],
      "source": [
        "#print(\"Classification Report\\n\")\n",
        "#print(classification_report(y_test,nb_pred))\n",
        "#print(\"Confusion Matrix\\n\")\n",
        "#print(confusion_matrix(y_test, nb_pred))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 169,
      "metadata": {
        "id": "hbeefd4VHtWP"
      },
      "outputs": [],
      "source": [
        "#from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, plot_roc_curve,roc_curve\n",
        "#print(\"===================Logistic Regression================\\n\")\n",
        "#print(\"LogReg Accuracy Score :\", (accuracy_score(y_test,LR_pred_test)))\n",
        "#print(\"LogReg Precision Score :\", (precision_score(y_test,LR_pred_test)))\n",
        "#print(\"LogReg Recall Score :\", (recall_score(y_test,LR_pred_test)))\n",
        "#print(\"LogReg F1 Score :\", (f1_score(y_test,LR_pred_test)))\n",
        "#print(\"LogReg ROC/AUC Score :\", (roc_auc_score(y_test,LR_pred_test)))\n",
        "#print(\"======================RFC=============================\\n\")\n",
        "#print(\"RFC Acccuracy Score : \", (accuracy_score(y_test,rfc_pred_test)))\n",
        "#print(\"RFC Precision Score : \", (precision_score(y_test, rfc_pred_test)))\n",
        "#print(\"RFC Recall Score : \", (recall_score(y_test, rfc_pred_test)))\n",
        "#print(\"RFC F1 Score : \", (f1_score(y_test, rfc_pred_test)))\n",
        "#print(\"RFC ROC/AUC Score : \", (roc_auc_score(y_test, rfc_pred_test)))\n",
        "#print(\"===================Decision Tree=====================\\n\")\n",
        "#print(\"DTree Accuracy Score : \", (accuracy_score(y_test, DT_pred_test)))\n",
        "#print(\"DTree Precision Score : \", (precision_score(y_test, DT_pred_test)))\n",
        "#print(\"DTree Recall Score : \", (recall_score(y_test, DT_pred_test)))\n",
        "#print(\"DTree F1 Score : \", (f1_score(y_test, DT_pred_test)))\n",
        "#print(\"DTree ROC/AUC Score : \", (roc_auc_score(y_test, DT_pred_test)))\n",
        "#print(\"===================Naive Bayes=====================\\n\")\n",
        "#print(\"NB Accuracy Score : \", (accuracy_score(y_test, nb_pred)))\n",
        "#print(\"NB Precision Score : \", (precision_score(y_test, nb_pred)))\n",
        "#print(\"NB Recall Score : \", (recall_score(y_test, nb_pred)))\n",
        "#print(\"NB F1 Score : \", (f1_score(y_test, nb_pred)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qk8kMGirfGzo"
      },
      "source": [
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3mrmWk0ofG2I"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "coSGWudqfG4w"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T09V-nW_fG8B"
      },
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.10.1 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.1"
    },
    "vscode": {
      "interpreter": {
        "hash": "2a78d80ae01c5bff69a96dcc1c54adfab24df5a769db2b5f1c4cbd7d56bb9eda"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}