project2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "toc_visible": true,
      "authorship_tag": "ABX9TyMpC4jPWuHJlpm1AUqb5aOL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Pralay-Kumar-Das/image_processing/blob/master/project2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%tensorflow_version 2.x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RU1n2sdTfX4I",
        "outputId": "bcbc418d-22ed-49c6-d7f4-13efbfdd9abd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab only includes TensorFlow 2.x; %tensorflow_version has no effect.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g80rZM0NBDJW"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget 'https://www.dropbox.com/sh/kpf9z73woodfssv/AAAw1_JIzpuVvwteJCma0xMla?dl=0'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qSmxn69YBXu3",
        "outputId": "cce1f0ec-9cfa-4553-a0a3-6115794e924e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-08-04 05:12:46--  https://www.dropbox.com/sh/kpf9z73woodfssv/AAAw1_JIzpuVvwteJCma0xMla?dl=0\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.1.18, 2620:100:6016:18::a27d:112\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.1.18|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /sh/raw/kpf9z73woodfssv/AAAw1_JIzpuVvwteJCma0xMla [following]\n",
            "--2023-08-04 05:12:46--  https://www.dropbox.com/sh/raw/kpf9z73woodfssv/AAAw1_JIzpuVvwteJCma0xMla\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com/zip_download_get/BkgOqwvmgEEuucr8O5zWf3NCoMVz2Ix6zgqUUDWJe_zSBDSlQOs1KHh5ic9oPq0ZuK6iozjjCTI_0Khe8cDvj0eVyP9LOO_uAIaPf9zOwdbV9A# [following]\n",
            "--2023-08-04 05:12:46--  https://uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com/zip_download_get/BkgOqwvmgEEuucr8O5zWf3NCoMVz2Ix6zgqUUDWJe_zSBDSlQOs1KHh5ic9oPq0ZuK6iozjjCTI_0Khe8cDvj0eVyP9LOO_uAIaPf9zOwdbV9A\n",
            "Resolving uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com (uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com)... 162.125.1.15, 2620:100:6016:15::a27d:10f\n",
            "Connecting to uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com (uc1b6f47a510f488961599473e86.dl.dropboxusercontent.com)|162.125.1.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 124207018 (118M) [application/zip]\n",
            "Saving to: ‘AAAw1_JIzpuVvwteJCma0xMla?dl=0.1’\n",
            "\n",
            "AAAw1_JIzpuVvwteJCm 100%[===================>] 118.45M   172MB/s    in 0.7s    \n",
            "\n",
            "2023-08-04 05:12:47 (172 MB/s) - ‘AAAw1_JIzpuVvwteJCma0xMla?dl=0.1’ saved [124207018/124207018]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip '/content/AAAw1_JIzpuVvwteJCma0xMla?dl=0'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1hyoqukf9c4J",
        "outputId": "b586fb12-5b8b-4d88-92ea-74098b83984f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/AAAw1_JIzpuVvwteJCma0xMla?dl=0\n",
            "warning:  stripped absolute path spec from /\n",
            "mapname:  conversion of  failed\n",
            " extracting: train.csv               \n",
            "\n",
            "1 archive had fatal errors.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.read_csv('/content/train.csv')"
      ],
      "metadata": {
        "id": "I4QhxYoWBd76"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "TdoY5RluBsmC",
        "outputId": "f048b0be-d9d7-4eb1-9fc5-748429188f10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          qid  \\\n",
              "0        00002165364db923c7e6   \n",
              "1        000032939017120e6e44   \n",
              "2        0000412ca6e4628ce2cf   \n",
              "3        000042bf85aa498cd78e   \n",
              "4        0000455dfa3e01eae3af   \n",
              "...                       ...   \n",
              "1306117  ffffcc4e2331aaf1e41e   \n",
              "1306118  ffffd431801e5a2f4861   \n",
              "1306119  ffffd48fb36b63db010c   \n",
              "1306120  ffffec519fa37cf60c78   \n",
              "1306121  ffffed09fedb5088744a   \n",
              "\n",
              "                                             question_text  target  \n",
              "0        How did Quebec nationalists see their province...       0  \n",
              "1        Do you have an adopted dog, how would you enco...       0  \n",
              "2        Why does velocity affect time? Does velocity a...       0  \n",
              "3        How did Otto von Guericke used the Magdeburg h...       0  \n",
              "4        Can I convert montra helicon D to a mountain b...       0  \n",
              "...                                                    ...     ...  \n",
              "1306117  What other technical skills do you need as a c...       0  \n",
              "1306118  Does MS in ECE have good job prospects in USA ...       0  \n",
              "1306119                          Is foam insulation toxic?       0  \n",
              "1306120  How can one start a research project based on ...       0  \n",
              "1306121  Who wins in a battle between a Wolverine and a...       0  \n",
              "\n",
              "[1306122 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-42765dad-fbbd-4481-9305-a19be42aede5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>question_text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00002165364db923c7e6</td>\n",
              "      <td>How did Quebec nationalists see their province...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000032939017120e6e44</td>\n",
              "      <td>Do you have an adopted dog, how would you enco...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0000412ca6e4628ce2cf</td>\n",
              "      <td>Why does velocity affect time? Does velocity a...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>000042bf85aa498cd78e</td>\n",
              "      <td>How did Otto von Guericke used the Magdeburg h...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0000455dfa3e01eae3af</td>\n",
              "      <td>Can I convert montra helicon D to a mountain b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306117</th>\n",
              "      <td>ffffcc4e2331aaf1e41e</td>\n",
              "      <td>What other technical skills do you need as a c...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306118</th>\n",
              "      <td>ffffd431801e5a2f4861</td>\n",
              "      <td>Does MS in ECE have good job prospects in USA ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306119</th>\n",
              "      <td>ffffd48fb36b63db010c</td>\n",
              "      <td>Is foam insulation toxic?</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306120</th>\n",
              "      <td>ffffec519fa37cf60c78</td>\n",
              "      <td>How can one start a research project based on ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306121</th>\n",
              "      <td>ffffed09fedb5088744a</td>\n",
              "      <td>Who wins in a battle between a Wolverine and a...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1306122 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-42765dad-fbbd-4481-9305-a19be42aede5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-21943f3e-0abf-46ad-8488-ce5911da2255\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-21943f3e-0abf-46ad-8488-ce5911da2255')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-21943f3e-0abf-46ad-8488-ce5911da2255 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-42765dad-fbbd-4481-9305-a19be42aede5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-42765dad-fbbd-4481-9305-a19be42aede5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['target'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x4FUfJdFB0Mh",
        "outputId": "321f7214-4907-41ce-f7d2-4ae3e42992a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    1225312\n",
              "1      80810\n",
              "Name: target, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.tokenize import word_tokenize,sent_tokenize"
      ],
      "metadata": {
        "id": "fcg4Dp-EB6tk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0fb2345e-0956-43cc-864e-67dc2c0d8b89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent_lens=[]\n",
        "for i in range(len(df)):\n",
        "    a=df.loc[i,'question_text']\n",
        "    try:\n",
        "        sent_lens.append(len(word_tokenize(a)))\n",
        "\n",
        "    except Exception as e:\n",
        "        continue\n",
        "\n",
        "max(sent_lens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5_nlHLqe2nL1",
        "outputId": "63f942de-d8ae-4ebb-b069-91783ec415da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "412"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.loc[1,'question_text']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "_d9VGcbH3JZy",
        "outputId": "5d83cf14-6023-4b28-bfd4-15b722590a35"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Do you have an adopted dog, how would you encourage people to adopt and not shop?'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "np.quantile(sent_lens,0.99),np.quantile(sent_lens,0.95),np.quantile(sent_lens,0.9)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5kBYg6rZ8YQh",
        "outputId": "89a52ccb-6d2d-44d3-8312-9a72997a91ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(45.0, 31.0, 25.0)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.quantile(sent_lens,1),np.quantile(sent_lens,0.999),np.quantile(sent_lens,0.9999)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zekWbctH-_8o",
        "outputId": "505c5f05-1b3b-4d92-9ea4-9abd42cac48c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(412, 56.0, 64.0)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "a=[]\n",
        "for i in sent_lens:\n",
        "    if i>100:\n",
        "        a.append(i)\n",
        "        print(i)\n",
        "print('.No. of sentences having 100  words or more are',len(a) )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KImCx7CI_HIk",
        "outputId": "b98f4d82-ea06-4f9f-b9a2-fb9b73a71956"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "361\n",
            "101\n",
            "155\n",
            "147\n",
            "182\n",
            "108\n",
            "412\n",
            "123\n",
            "177\n",
            "125\n",
            "111\n",
            ".No. of sentences having 100  words or more are 11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "max_len=80\n"
      ],
      "metadata": {
        "id": "VybaZpi2_W5U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.utils import pad_sequences\n",
        "from keras.models import Sequential,Model\n",
        "from keras.layers import Dense,LSTM,Embedding,Dropout,Input"
      ],
      "metadata": {
        "id": "TudA4889_9hN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X=df['question_text']\n",
        "y=df['target']\n",
        "x_train,x_test,y_train,y_test=train_test_split(X,y,test_size=0.2)"
      ],
      "metadata": {
        "id": "zMQ4UB6jBZcx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tok=Tokenizer(char_level=False,split=' ')\n",
        "tok.fit_on_texts(x_train)\n",
        "seq=tok.texts_to_sequences(x_train)\n",
        "x_train_seq=pad_sequences(seq,maxlen=max_len)"
      ],
      "metadata": {
        "id": "Lm7fg-2HBZtv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "seq1=tok.texts_to_sequences(x_test)\n",
        "x_test_seq=pad_sequences(seq1,maxlen=max_len)"
      ],
      "metadata": {
        "id": "cFJnVYU6BZ53"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train_seq.shape,x_test_seq.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUaIrN5yDL7R",
        "outputId": "44ae8741-8211-4529-a05d-1e9f6e4b7bd1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1044897, 80), (261225, 80))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_size=len(tok.word_index)\n",
        "vocab_size"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vKkdvgfYDLW9",
        "outputId": "3e0ed60a-9d62-450e-ea1e-dcd16eaf0842"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "195852"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget 'http://nlp.stanford.edu/data/glove.42B.300d.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xh676F0aA6Th",
        "outputId": "4a1019d2-2839-4f87-caf0-47c8d438bfa6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-08-04 09:46:31--  http://nlp.stanford.edu/data/glove.42B.300d.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.42B.300d.zip [following]\n",
            "--2023-08-04 09:46:31--  https://nlp.stanford.edu/data/glove.42B.300d.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: https://downloads.cs.stanford.edu/nlp/data/glove.42B.300d.zip [following]\n",
            "--2023-08-04 09:46:31--  https://downloads.cs.stanford.edu/nlp/data/glove.42B.300d.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1877800501 (1.7G) [application/zip]\n",
            "Saving to: ‘glove.42B.300d.zip.1’\n",
            "\n",
            "glove.42B.300d.zip. 100%[===================>]   1.75G  5.06MB/s    in 5m 53s  \n",
            "\n",
            "2023-08-04 09:52:24 (5.07 MB/s) - ‘glove.42B.300d.zip.1’ saved [1877800501/1877800501]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip '/content/glove.42B.300d.zip.1'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NpGN3tL5D36M",
        "outputId": "61a24bbd-e8d8-41ba-ff40-a15607ce409d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/glove.42B.300d.zip.1\n",
            "  inflating: glove.42B.300d.txt      \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embeding_index={}\n",
        "\n",
        "f=open('/content/glove.42B.300d.txt',encoding='utf-8')\n",
        "\n",
        "for line in f:\n",
        "    values=line.split()\n",
        "    word=values[0]\n",
        "    mtx=np.asarray(values[1:],dtype='float32')\n",
        "    embeding_index[word]=mtx\n",
        "f.close()"
      ],
      "metadata": {
        "id": "bK-05yO7BRdY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm '/content/glove.42B.300d.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lI8X3hPrEAmy",
        "outputId": "e24e1987-55b9-4f40-bba1-1c2675f5372c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "rm: cannot remove '/content/glove.42B.300d.zip': No such file or directory\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix=np.zeros((vocab_size+1,300))\n",
        "not_available=0\n",
        "for word,i in tok.word_index.items():\n",
        "    embed_vector=embeding_index.get(word)\n",
        "    if embed_vector is not None:\n",
        "        embedding_matrix[i]=embed_vector\n",
        "\n",
        "    else:\n",
        "        not_available+=1\n",
        "not_available"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nsdd_mVQEN8t",
        "outputId": "6552e2b4-18f3-44c2-d011-d0b61f88bad3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "61908"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix.shape,embedding_matrix"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8m60k3riFRUf",
        "outputId": "ce887802-bd90-4e6d-ffcb-89f1fe2f44a4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((195853, 300),\n",
              " array([[ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "          0.        ,  0.        ],\n",
              "        [-0.20838   , -0.14932001, -0.017528  , ..., -0.54066002,\n",
              "          0.21199   , -0.0094357 ],\n",
              "        [ 0.13877   , -0.0021201 , -0.091039  , ...,  0.024614  ,\n",
              "          0.46514001,  0.11363   ],\n",
              "        ...,\n",
              "        [-0.2922    , -0.20874999,  0.17894   , ...,  0.39546999,\n",
              "          0.10891   , -0.041327  ],\n",
              "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "          0.        ,  0.        ],\n",
              "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "          0.        ,  0.        ]]))"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model=Sequential()\n",
        "model.add((Embedding(input_dim=(vocab_size+1),output_dim=300,input_length=max_len,\n",
        "                          mask_zero=True,\n",
        "                          weights=[embedding_matrix],trainable=False)))\n",
        "model.add(LSTM(64,return_sequences=True))\n",
        "model.add(LSTM(32))\n",
        "model.add(Dense(25,activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(16,activation='relu'))\n",
        "model.add(Dropout(0.3))\n",
        "model.add(Dense(1,activation='sigmoid'))\n",
        "model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZWpdlex0dg28",
        "outputId": "f288641d-b724-4482-d034-edec6bb2eaf6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 80, 300)           58755900  \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 80, 64)            93440     \n",
            "                                                                 \n",
            " lstm_1 (LSTM)               (None, 32)                12416     \n",
            "                                                                 \n",
            " dense (Dense)               (None, 25)                825       \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 25)                0         \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 16)                416       \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 16)                0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 58,863,014\n",
            "Trainable params: 107,114\n",
            "Non-trainable params: 58,755,900\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(x_train_seq,y_train,\n",
        "          validation_data=[x_test_seq,y_test],epochs=6,batch_size=2000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MQkAXpeYdgJA",
        "outputId": "edb558d6-6d37-4e4e-8e8b-ecdc91950341"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/6\n",
            "523/523 [==============================] - 365s 670ms/step - loss: 0.1837 - accuracy: 0.9401 - val_loss: 0.1180 - val_accuracy: 0.9530\n",
            "Epoch 2/6\n",
            "523/523 [==============================] - 261s 499ms/step - loss: 0.1224 - accuracy: 0.9541 - val_loss: 0.1100 - val_accuracy: 0.9560\n",
            "Epoch 3/6\n",
            "523/523 [==============================] - 264s 505ms/step - loss: 0.1138 - accuracy: 0.9566 - val_loss: 0.1069 - val_accuracy: 0.9576\n",
            "Epoch 4/6\n",
            "523/523 [==============================] - 259s 496ms/step - loss: 0.1089 - accuracy: 0.9582 - val_loss: 0.1053 - val_accuracy: 0.9577\n",
            "Epoch 5/6\n",
            "523/523 [==============================] - 251s 479ms/step - loss: 0.1053 - accuracy: 0.9590 - val_loss: 0.1052 - val_accuracy: 0.9577\n",
            "Epoch 6/6\n",
            "523/523 [==============================] - 264s 505ms/step - loss: 0.1024 - accuracy: 0.9596 - val_loss: 0.1040 - val_accuracy: 0.9578\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7d58c3d0f160>"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(x_train_seq,y_train,\n",
        "          validation_data=[x_test_seq,y_test],epochs=6,batch_size=20000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pJviUbfYFrf_",
        "outputId": "74bf5c60-87f1-4a0b-a285-918ceb4e4ed0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/6\n",
            "53/53 [==============================] - 45s 841ms/step - loss: 0.0978 - accuracy: 0.9611 - val_loss: 0.1052 - val_accuracy: 0.9589\n",
            "Epoch 2/6\n",
            "53/53 [==============================] - 44s 836ms/step - loss: 0.0961 - accuracy: 0.9616 - val_loss: 0.1059 - val_accuracy: 0.9587\n",
            "Epoch 3/6\n",
            "53/53 [==============================] - 44s 833ms/step - loss: 0.0958 - accuracy: 0.9619 - val_loss: 0.1053 - val_accuracy: 0.9584\n",
            "Epoch 4/6\n",
            "53/53 [==============================] - 44s 835ms/step - loss: 0.0951 - accuracy: 0.9619 - val_loss: 0.1055 - val_accuracy: 0.9583\n",
            "Epoch 5/6\n",
            "53/53 [==============================] - 44s 832ms/step - loss: 0.0946 - accuracy: 0.9620 - val_loss: 0.1055 - val_accuracy: 0.9583\n",
            "Epoch 6/6\n",
            "53/53 [==============================] - 44s 837ms/step - loss: 0.0940 - accuracy: 0.9623 - val_loss: 0.1057 - val_accuracy: 0.9585\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7d589ce33400>"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import metrics as m\n",
        "pred=model.predict(x_test_seq)\n",
        "print('roc_auc_score',m.roc_auc_score(y_test,pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-oPe0fhQdfav",
        "outputId": "4600a0fc-dc57-413e-8715-a2d0dafde65a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8164/8164 [==============================] - 279s 34ms/step\n",
            "roc_auc_score 0.9639209209071631\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predicts=np.where(pred>0.5,1,0)\n",
        "print('classification report\\n',m.classification_report(y_test,predicts))\n",
        "print('confusion_matrix\\n',m.confusion_matrix(y_test,predicts))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "THNZyScrmBBi",
        "outputId": "6f55698a-8324-4636-f297-0723cbc82876"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "classification report\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.98      0.98    245042\n",
            "           1       0.67      0.64      0.66     16183\n",
            "\n",
            "    accuracy                           0.96    261225\n",
            "   macro avg       0.82      0.81      0.82    261225\n",
            "weighted avg       0.96      0.96      0.96    261225\n",
            "\n",
            "confusion_matrix\n",
            " [[239990   5052]\n",
            " [  5780  10403]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget 'http://vectors.nlpl.eu/repository/20/0.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s2n1GcmvFD6c",
        "outputId": "6e1aec5b-44ab-4193-f1de-8386d620282f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-08-04 08:27:35--  http://vectors.nlpl.eu/repository/20/0.zip\n",
            "Resolving vectors.nlpl.eu (vectors.nlpl.eu)... 129.240.189.181\n",
            "Connecting to vectors.nlpl.eu (vectors.nlpl.eu)|129.240.189.181|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 344050746 (328M) [application/zip]\n",
            "Saving to: ‘0.zip’\n",
            "\n",
            "0.zip               100%[===================>] 328.11M  18.4MB/s    in 18s     \n",
            "\n",
            "2023-08-04 08:27:55 (17.8 MB/s) - ‘0.zip’ saved [344050746/344050746]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip '/content/0.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EhTwQhiOFhbB",
        "outputId": "a79b2cbe-b02c-4b0c-aa23-2ae1bde41bfc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/0.zip\n",
            "  inflating: meta.json               \n",
            "  inflating: model.bin               \n",
            "  inflating: model.txt               \n",
            "  inflating: README                  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_index1={}\n",
        "f=open('/content/model.txt',encoding='utf-8')\n",
        "for i,line in enumerate(f):\n",
        "    if i==0:\n",
        "        continue\n",
        "    values=line.split()\n",
        "    words=values[0].split('_')[0]\n",
        "    matrix=np.asarray(values[1:],dtype='float64')\n",
        "    embedding_index1[words]=matrix\n",
        "\n",
        "f.close()\n",
        "#print(embedding_index1)"
      ],
      "metadata": {
        "id": "-EeBrZYXFqPe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix1=np.zeros((vocab_size+1,300))\n",
        "not_avail=0\n",
        "for word,matrix in tok.word_index.items():\n",
        "    em_vec=embedding_index1.get(word)\n",
        "    if em_vec is not None:\n",
        "        embedding_matrix1[matrix]=em_vec\n",
        "    else:\n",
        "        not_avail+=1\n",
        "\n",
        "print(not_avail)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5qZbi2_tHlYz",
        "outputId": "ab3faf5d-f0bf-43b4-d820-68e9ea78a922"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "155564\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix1.shape,embedding_matrix.shape,not_available,not_avail"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mtmH7KhlKfZ2",
        "outputId": "67e80c9e-c8cb-4362-c40e-95f6a752f900"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((196049, 300), (196049, 300), 62121, 155564)"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1=Sequential()\n",
        "model1.add((Embedding(input_dim=(vocab_size+1),output_dim=300,input_length=max_len,\n",
        "                          mask_zero=True,\n",
        "                          weights=[embedding_matrix1],trainable=False)))\n",
        "model1.add(LSTM(64,return_sequences=True))\n",
        "model1.add(LSTM(32))\n",
        "model1.add(Dense(25,activation='relu'))\n",
        "model1.add(Dropout(0.5))\n",
        "model1.add(Dense(16,activation='relu'))\n",
        "model1.add(Dropout(0.3))\n",
        "model1.add(Dense(8,activation='relu'))\n",
        "model1.add(Dense(1,activation='sigmoid'))\n",
        "model1.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])\n",
        "print(model1.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gx-l8UqErOiG",
        "outputId": "0a8d86c2-6c69-497b-8c22-5bcba504e84d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_4 (Embedding)     (None, 80, 300)           58814700  \n",
            "                                                                 \n",
            " lstm_6 (LSTM)               (None, 80, 64)            93440     \n",
            "                                                                 \n",
            " lstm_7 (LSTM)               (None, 32)                12416     \n",
            "                                                                 \n",
            " dense_10 (Dense)            (None, 25)                825       \n",
            "                                                                 \n",
            " dropout_4 (Dropout)         (None, 25)                0         \n",
            "                                                                 \n",
            " dense_11 (Dense)            (None, 16)                416       \n",
            "                                                                 \n",
            " dropout_5 (Dropout)         (None, 16)                0         \n",
            "                                                                 \n",
            " dense_12 (Dense)            (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 58,921,814\n",
            "Trainable params: 107,114\n",
            "Non-trainable params: 58,814,700\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.fit(x_train_seq,y_train,\n",
        "          validation_data=[x_test_seq,y_test],epochs=6,batch_size=2000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dRTRoxfhrOUf",
        "outputId": "e598da7e-d206-4f43-f814-c62adbd981b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/6\n",
            "523/523 [==============================] - 271s 501ms/step - loss: 0.1574 - accuracy: 0.9439 - val_loss: 0.1201 - val_accuracy: 0.9521\n",
            "Epoch 2/6\n",
            "523/523 [==============================] - 253s 483ms/step - loss: 0.1221 - accuracy: 0.9518 - val_loss: 0.1113 - val_accuracy: 0.9558\n",
            "Epoch 3/6\n",
            "523/523 [==============================] - 252s 482ms/step - loss: 0.1155 - accuracy: 0.9539 - val_loss: 0.1110 - val_accuracy: 0.9574\n",
            "Epoch 4/6\n",
            "523/523 [==============================] - 248s 474ms/step - loss: 0.1108 - accuracy: 0.9554 - val_loss: 0.1115 - val_accuracy: 0.9577\n",
            "Epoch 5/6\n",
            "523/523 [==============================] - 245s 469ms/step - loss: 0.1067 - accuracy: 0.9571 - val_loss: 0.1086 - val_accuracy: 0.9584\n",
            "Epoch 6/6\n",
            "523/523 [==============================] - 244s 467ms/step - loss: 0.1033 - accuracy: 0.9583 - val_loss: 0.1052 - val_accuracy: 0.9582\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7994a61590c0>"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred1=model1.predict(x_test_seq)\n",
        "print('roc_auc_score',m.roc_auc_score(y_test,pred1))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8pQA7nmtrNey",
        "outputId": "59af8bab-5ee7-4bf5-c0d9-4c458565c7ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8164/8164 [==============================] - 295s 36ms/step\n",
            "roc_auc_score 0.9638871927831558\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predicts1=np.where(pred1>0.5,1,0)\n",
        "print('classification report\\n',m.classification_report(y_test,predicts1))\n",
        "print('confusion_matrix\\n',m.confusion_matrix(y_test,predicts1))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_H-KEuJ9rLyD",
        "outputId": "e23954df-7087-4490-bc19-dc08d6ddf1a6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "classification report\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.98      0.98    244913\n",
            "           1       0.67      0.65      0.66     16312\n",
            "\n",
            "    accuracy                           0.96    261225\n",
            "   macro avg       0.82      0.81      0.82    261225\n",
            "weighted avg       0.96      0.96      0.96    261225\n",
            "\n",
            "confusion_matrix\n",
            " [[239733   5180]\n",
            " [  5744  10568]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LkDYeJtJJrcv",
        "outputId": "fddcbece-9280-4c28-a89f-e6023ff97467"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(261225, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "final_model=model\n",
        "pred=final_model.predict(x_test_seq)\n",
        "#predictions=pd.DataFrame({'target':pred})\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5Y4MpjMrK_N",
        "outputId": "8427676a-eded-49b2-bf9d-64cbb5d401bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8164/8164 [==============================] - 315s 39ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred.shape,pred.reshape(-1,1),pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxHQX33cL9K9",
        "outputId": "626e9fb2-15b5-4273-a552-631e8d609fb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((261225, 1),\n",
              " array([[4.6802495e-09],\n",
              "        [1.8024726e-05],\n",
              "        [4.9786445e-01],\n",
              "        ...,\n",
              "        [4.6823923e-09],\n",
              "        [1.0332413e-02],\n",
              "        [1.5137845e-04]], dtype=float32),\n",
              " array([[4.6802495e-09],\n",
              "        [1.8024726e-05],\n",
              "        [4.9786445e-01],\n",
              "        ...,\n",
              "        [4.6823923e-09],\n",
              "        [1.0332413e-02],\n",
              "        [1.5137845e-04]], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions=pd.DataFrame({'target':pred.flatten()})"
      ],
      "metadata": {
        "id": "4rGxb7H6L-Nr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.to_csv('Pralay_kumar_das_spam_filter.csv',index=False)"
      ],
      "metadata": {
        "id": "pKM0YFyB6IYu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "jiRf7WwrONyg"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}