MovieSentiment_allennlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "deepCognitoMovieSentiment_allennlp.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "HqA2-Ysqu7lw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3690
        },
        "outputId": "c0498ef1-911b-4d20-cda1-6c4775228341"
      },
      "cell_type": "code",
      "source": [
        "!pip install overrides\n",
        "!pip install allennlp"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting overrides\n",
            "  Downloading https://files.pythonhosted.org/packages/de/55/3100c6d14c1ed177492fcf8f07c4a7d2d6c996c0a7fc6a9a0a41308e7eec/overrides-1.9.tar.gz\n",
            "Building wheels for collected packages: overrides\n",
            "  Building wheel for overrides (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/8d/52/86/e5a83b1797e7d263b458d2334edd2704c78508b3eea9323718\n",
            "Successfully built overrides\n",
            "Installing collected packages: overrides\n",
            "Successfully installed overrides-1.9\n",
            "Collecting allennlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/32/d6d0a93a23763f366df2dbd4e007e45ce4d2ad97e6315506db9da8af7731/allennlp-0.8.2-py3-none-any.whl (5.6MB)\n",
            "\u001b[K    100% |████████████████████████████████| 5.6MB 7.4MB/s \n",
            "\u001b[?25hCollecting tensorboardX==1.2 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c5/22/43f4f0318f7c68a1000dbb700a353b745584bc2397437832d15ba69ea5f1/tensorboardX-1.2-py2.py3-none-any.whl (44kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 19.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: flask==1.0.2 in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.0.2)\n",
            "Requirement already satisfied: requests>=2.18 in /usr/local/lib/python3.6/dist-packages (from allennlp) (2.18.4)\n",
            "Collecting responses>=0.7 (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/2f/d8/a77cbd4cb8366ad0e275f2c642b50b401da22a2f5714e003e499fddca106/responses-0.10.5-py2.py3-none-any.whl\n",
            "Collecting moto==1.3.4 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/8f/7b36e81ff067d0e7bf90f7210b351c0cfe6657f79fa4dcb0cb4787462e05/moto-1.3.4-py2.py3-none-any.whl (548kB)\n",
            "\u001b[K    100% |████████████████████████████████| 552kB 24.5MB/s \n",
            "\u001b[?25hCollecting jsonnet==0.10.0; sys_platform != \"win32\" (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3c/83/d49904ee98dd4fbba6a003938e30e76251951c4bdb49628b4f92e5009a42/jsonnet-0.10.0.tar.gz (124kB)\n",
            "\u001b[K    100% |████████████████████████████████| 133kB 30.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from allennlp) (3.2.5)\n",
            "Collecting pytz==2017.3 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/7f/e7d1acbd433b929168a4fb4182a2ff3c33653717195a26c1de099ad1ef29/pytz-2017.3-py2.py3-none-any.whl (511kB)\n",
            "\u001b[K    100% |████████████████████████████████| 512kB 23.7MB/s \n",
            "\u001b[?25hCollecting gevent==1.3.6 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/43/3d/a19fece28ba1b5133cf74bd22a229d77b4d9cc4b24aa8f263cca2845c555/gevent-1.3.6-cp36-cp36m-manylinux1_x86_64.whl (4.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 4.5MB 7.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from allennlp) (2.8.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.9.106)\n",
            "Collecting matplotlib==2.2.3 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/59/f235ab21bbe7b7c6570c4abf17ffb893071f4fa3b9cf557b09b60359ad9a/matplotlib-2.2.3-cp36-cp36m-manylinux1_x86_64.whl (12.6MB)\n",
            "\u001b[K    100% |████████████████████████████████| 12.6MB 3.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from allennlp) (0.20.3)\n",
            "Collecting pytorch-pretrained-bert>=0.6.0 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5d/3c/d5fa084dd3a82ffc645aba78c417e6072ff48552e3301b1fa3bd711e03d4/pytorch_pretrained_bert-0.6.1-py3-none-any.whl (114kB)\n",
            "\u001b[K    100% |████████████████████████████████| 122kB 32.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.0.1.post2)\n",
            "Collecting parsimonious==0.8.0 (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/4a/89/32c55944cd30dff856f16859ee325b13c83c260d0c56c0eed511e8063c87/parsimonious-0.8.0.tar.gz\n",
            "Collecting unidecode (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/31/39/53096f9217b057cb049fe872b7fc7ce799a1a89b76cf917d9639e7a558b5/Unidecode-1.0.23-py2.py3-none-any.whl (237kB)\n",
            "\u001b[K    100% |████████████████████████████████| 245kB 26.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.6/dist-packages (from allennlp) (3.6.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.14.6)\n",
            "Requirement already satisfied: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from allennlp) (0.5.6)\n",
            "Collecting flask-cors==3.0.7 (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/65/cb/683f71ff8daa3aea0a5cbb276074de39f9ab66d3fbb8ad5efb5bb83e90d2/Flask_Cors-3.0.7-py2.py3-none-any.whl\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.1.0)\n",
            "Collecting ftfy (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8f/86/df789c5834f15ae1ca53a8d4c1fc4788676c2e32112f6a786f2625d9c6e6/ftfy-5.5.1-py3-none-any.whl (43kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 21.1MB/s \n",
            "\u001b[?25hCollecting numpydoc==0.8.0 (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/95/a8/b4706a6270f0475541c5c1ee3373c7a3b793936ec1f517f1a1dab4f896c0/numpydoc-0.8.0.tar.gz\n",
            "Collecting flaky (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/02/42/cca66659a786567c8af98587d66d75e7d2b6e65662f8daab75db708ac35b/flaky-3.5.3-py2.py3-none-any.whl\n",
            "Requirement already satisfied: tqdm>=4.19 in /usr/local/lib/python3.6/dist-packages (from allennlp) (4.28.1)\n",
            "Collecting awscli>=1.11.91 (from allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/16/d1/43232fc0661b771c3a9a29c6518263217b2966a506b871cc0290c3e52c62/awscli-1.16.121-py2.py3-none-any.whl (1.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.5MB 16.1MB/s \n",
            "\u001b[?25hCollecting conllu==0.11 (from allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/d4/2c/856344d9b69baf5b374c395b4286626181a80f0c2b2f704914d18a1cea47/conllu-0.11-py2.py3-none-any.whl\n",
            "Requirement already satisfied: editdistance in /usr/local/lib/python3.6/dist-packages (from allennlp) (0.5.2)\n",
            "Requirement already satisfied: overrides in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.9)\n",
            "Requirement already satisfied: spacy<2.1,>=2.0 in /usr/local/lib/python3.6/dist-packages (from allennlp) (2.0.18)\n",
            "Requirement already satisfied: sqlparse==0.2.4 in /usr/local/lib/python3.6/dist-packages (from allennlp) (0.2.4)\n",
            "Requirement already satisfied: protobuf>=0.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorboardX==1.2->allennlp) (3.6.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX==1.2->allennlp) (1.11.0)\n",
            "Requirement already satisfied: click>=5.1 in /usr/local/lib/python3.6/dist-packages (from flask==1.0.2->allennlp) (7.0)\n",
            "Requirement already satisfied: Werkzeug>=0.14 in /usr/local/lib/python3.6/dist-packages (from flask==1.0.2->allennlp) (0.14.1)\n",
            "Requirement already satisfied: Jinja2>=2.10 in /usr/local/lib/python3.6/dist-packages (from flask==1.0.2->allennlp) (2.10)\n",
            "Requirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask==1.0.2->allennlp) (1.1.0)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp) (1.22)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp) (3.0.4)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp) (2.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp) (2018.11.29)\n",
            "Requirement already satisfied: boto>=2.36.0 in /usr/local/lib/python3.6/dist-packages (from moto==1.3.4->allennlp) (2.49.0)\n",
            "Requirement already satisfied: botocore>=1.9.16 in /usr/local/lib/python3.6/dist-packages (from moto==1.3.4->allennlp) (1.12.106)\n",
            "Requirement already satisfied: mock in /usr/local/lib/python3.6/dist-packages (from moto==1.3.4->allennlp) (2.0.0)\n",
            "Collecting docker>=2.5.1 (from moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/3c/b610f22b170b0f8fe4d8f78974878e116562389f666f99e6549567eb9d87/docker-3.7.0-py2.py3-none-any.whl (133kB)\n",
            "\u001b[K    100% |████████████████████████████████| 143kB 32.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from moto==1.3.4->allennlp) (2.5.3)\n",
            "Collecting cryptography>=2.0.0 (from moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5b/12/b0409a94dad366d98a8eee2a77678c7a73aafd8c0e4b835abea634ea3896/cryptography-2.6.1-cp34-abi3-manylinux1_x86_64.whl (2.3MB)\n",
            "\u001b[K    100% |████████████████████████████████| 2.3MB 13.6MB/s \n",
            "\u001b[?25hCollecting python-jose<3.0.0 (from moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/bf/5c/5fa238c0c5b0656994b52721dd8b1d7bf52ebd8786518dde794f44de86b6/python_jose-2.0.2-py2.py3-none-any.whl\n",
            "Collecting xmltodict (from moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/28/fd/30d5c1d3ac29ce229f6bdc40bbc20b28f716e8b363140c26eff19122d8a5/xmltodict-0.12.0-py2.py3-none-any.whl\n",
            "Collecting aws-xray-sdk<0.96,>=0.93 (from moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a4/a5/da7887285564f9e0ae5cd25a453cca36e2cd43d8ccc9effde260b4d80904/aws_xray_sdk-0.95-py2.py3-none-any.whl (52kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 24.4MB/s \n",
            "\u001b[?25hCollecting cookies (from moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6a/60/557f84aa2db629e5124aa05408b975b1b5d0e1cec16cde0bfa06aae097d3/cookies-2.2.1-py2.py3-none-any.whl (44kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 17.9MB/s \n",
            "\u001b[?25hCollecting jsondiff==1.1.1 (from moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/bd/5f/13e28a2f9abeda2ffb3f44f2f809b01b52bc02cdb63816e05b8c9cbbdfc5/jsondiff-1.1.1.tar.gz\n",
            "Collecting pyaml (from moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/c5/e1/1523fb1dab744e2c6b1f02446f2139a78726c18c062a8ddd53875abb20f8/pyaml-18.11.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: greenlet>=0.4.14; platform_python_implementation == \"CPython\" in /usr/local/lib/python3.6/dist-packages (from gevent==1.3.6->allennlp) (0.4.15)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->allennlp) (0.2.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->allennlp) (0.9.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib==2.2.3->allennlp) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==2.2.3->allennlp) (2.3.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==2.2.3->allennlp) (1.0.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert>=0.6.0->allennlp) (2018.1.10)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (1.8.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (6.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (40.8.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (0.7.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (19.1.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp) (1.3.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from ftfy->allennlp) (0.1.7)\n",
            "Requirement already satisfied: sphinx>=1.2.3 in /usr/local/lib/python3.6/dist-packages (from numpydoc==0.8.0->allennlp) (1.8.4)\n",
            "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from awscli>=1.11.91->allennlp) (0.14)\n",
            "Requirement already satisfied: PyYAML<=3.13,>=3.10 in /usr/local/lib/python3.6/dist-packages (from awscli>=1.11.91->allennlp) (3.13)\n",
            "Collecting rsa<=3.5.0,>=3.1.2 (from awscli>=1.11.91->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e1/ae/baedc9cb175552e95f3395c43055a6a5e125ae4d48a1d7a924baca83e92e/rsa-3.4.2-py2.py3-none-any.whl (46kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 21.4MB/s \n",
            "\u001b[?25hCollecting colorama<=0.3.9,>=0.2.5 (from awscli>=1.11.91->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/db/c8/7dcf9dbcb22429512708fe3a547f8b6101c0d02137acbd892505aee57adf/colorama-0.3.9-py2.py3-none-any.whl\n",
            "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (2.0.1)\n",
            "Requirement already satisfied: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (6.12.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (1.0.2)\n",
            "Requirement already satisfied: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (0.2.9)\n",
            "Requirement already satisfied: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (1.35)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (2.0.2)\n",
            "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy<2.1,>=2.0->allennlp) (0.9.6)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10->flask==1.0.2->allennlp) (1.1.1)\n",
            "Requirement already satisfied: pbr>=0.11 in /usr/local/lib/python3.6/dist-packages (from mock->moto==1.3.4->allennlp) (5.1.3)\n",
            "Collecting websocket-client>=0.32.0 (from docker>=2.5.1->moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/38/54/684db2ba1b7a203602808446b8686ee786f93b4a7e080cdc440cc7e06e56/websocket_client-0.55.0-py2.py3-none-any.whl (200kB)\n",
            "\u001b[K    100% |████████████████████████████████| 204kB 29.2MB/s \n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from docker>=2.5.1->moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Collecting asn1crypto>=0.21.0 (from cryptography>=2.0.0->moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/cd/35485615f45f30a510576f1a56d1e0a7ad7bd8ab5ed7cdc600ef7cd06222/asn1crypto-0.24.0-py2.py3-none-any.whl (101kB)\n",
            "\u001b[K    100% |████████████████████████████████| 102kB 30.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi!=1.11.3,>=1.8 in /usr/local/lib/python3.6/dist-packages (from cryptography>=2.0.0->moto==1.3.4->allennlp) (1.12.2)\n",
            "Collecting pycryptodome<4.0.0,>=3.3.1 (from python-jose<3.0.0->moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6d/cf/4b66bf1ac2484ca39599b4576d681186b61b543c2d2c29f9aa4ba3cc53b5/pycryptodome-3.7.3-cp36-cp36m-manylinux1_x86_64.whl (7.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 7.5MB 6.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: future<1.0 in /usr/local/lib/python3.6/dist-packages (from python-jose<3.0.0->moto==1.3.4->allennlp) (0.16.0)\n",
            "Collecting ecdsa<1.0 (from python-jose<3.0.0->moto==1.3.4->allennlp)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/63/f4/73669d51825516ce8c43b816c0a6b64cd6eb71d08b99820c00792cb42222/ecdsa-0.13-py2.py3-none-any.whl (86kB)\n",
            "\u001b[K    100% |████████████████████████████████| 92kB 28.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from aws-xray-sdk<0.96,>=0.93->moto==1.3.4->allennlp) (1.10.11)\n",
            "Collecting jsonpickle (from aws-xray-sdk<0.96,>=0.93->moto==1.3.4->allennlp)\n",
            "  Downloading https://files.pythonhosted.org/packages/dc/12/8c44eabb501e2bc0aec0dd152b328074d98a50968d3a02be28f6037f0c6a/jsonpickle-1.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (1.2.1)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (0.7.12)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (2.1.3)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (1.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (19.0)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (2.6.0)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.2.3->numpydoc==0.8.0->allennlp) (1.1.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<=3.5.0,>=3.1.2->awscli>=1.11.91->allennlp) (0.4.5)\n",
            "Requirement already satisfied: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy<2.1,>=2.0->allennlp) (0.4.3.2)\n",
            "Requirement already satisfied: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy<2.1,>=2.0->allennlp) (0.9.0.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi!=1.11.3,>=1.8->cryptography>=2.0.0->moto==1.3.4->allennlp) (2.19)\n",
            "Requirement already satisfied: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy<2.1,>=2.0->allennlp) (0.9.0)\n",
            "Building wheels for collected packages: jsonnet, parsimonious, numpydoc, jsondiff\n",
            "  Building wheel for jsonnet (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/a0/aa/f0/b4ab8854cf00f922a87787425cfbb789aac01ab2c2cd1b4ca4\n",
            "  Building wheel for parsimonious (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/bb/51/82/ae9b22a790f11e7be918939d01aa397c545ebb3723453c5fb4\n",
            "  Building wheel for numpydoc (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/ea/55/7f/3e25d754760ccd62d6796e5b2cfe25629346f52ea00753d549\n",
            "  Building wheel for jsondiff (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/68/08/07/69d839606fb7fdc778fa86476abc0a864693d45969a0c1936c\n",
            "Successfully built jsonnet parsimonious numpydoc jsondiff\n",
            "\u001b[31mimgaug 0.2.8 has requirement numpy>=1.15.0, but you'll have numpy 1.14.6 which is incompatible.\u001b[0m\n",
            "\u001b[31mfeaturetools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mfastai 1.0.46 has requirement numpy>=1.15, but you'll have numpy 1.14.6 which is incompatible.\u001b[0m\n",
            "\u001b[31malbumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.8 which is incompatible.\u001b[0m\n",
            "\u001b[31mawscli 1.16.121 has requirement botocore==1.12.111, but you'll have botocore 1.12.106 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tensorboardX, responses, pytz, websocket-client, docker-pycreds, docker, asn1crypto, cryptography, pycryptodome, ecdsa, python-jose, xmltodict, jsonpickle, aws-xray-sdk, cookies, jsondiff, pyaml, moto, jsonnet, gevent, matplotlib, pytorch-pretrained-bert, parsimonious, unidecode, flask-cors, ftfy, numpydoc, flaky, rsa, colorama, awscli, conllu, allennlp\n",
            "  Found existing installation: pytz 2018.9\n",
            "    Uninstalling pytz-2018.9:\n",
            "      Successfully uninstalled pytz-2018.9\n",
            "  Found existing installation: gevent 1.4.0\n",
            "    Uninstalling gevent-1.4.0:\n",
            "      Successfully uninstalled gevent-1.4.0\n",
            "  Found existing installation: matplotlib 3.0.3\n",
            "    Uninstalling matplotlib-3.0.3:\n",
            "      Successfully uninstalled matplotlib-3.0.3\n",
            "  Found existing installation: rsa 4.0\n",
            "    Uninstalling rsa-4.0:\n",
            "      Successfully uninstalled rsa-4.0\n",
            "Successfully installed allennlp-0.8.2 asn1crypto-0.24.0 aws-xray-sdk-0.95 awscli-1.16.121 colorama-0.3.9 conllu-0.11 cookies-2.2.1 cryptography-2.6.1 docker-3.7.0 docker-pycreds-0.4.0 ecdsa-0.13 flaky-3.5.3 flask-cors-3.0.7 ftfy-5.5.1 gevent-1.3.6 jsondiff-1.1.1 jsonnet-0.10.0 jsonpickle-1.1 matplotlib-2.2.3 moto-1.3.4 numpydoc-0.8.0 parsimonious-0.8.0 pyaml-18.11.0 pycryptodome-3.7.3 python-jose-2.0.2 pytorch-pretrained-bert-0.6.1 pytz-2017.3 responses-0.10.5 rsa-3.4.2 tensorboardX-1.2 unidecode-1.0.23 websocket-client-0.55.0 xmltodict-0.12.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "matplotlib",
                  "mpl_toolkits",
                  "pytz",
                  "rsa"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "D9gqVMK9ryvx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f0ca2acf-5837-45e7-8516-a6671083790e"
      },
      "cell_type": "code",
      "source": [
        "from typing import Iterator, List, Dict\n",
        "import logging, csv, re\n",
        "import torch\n",
        "from torchsummary import summary\n",
        "import torchtext\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm_notebook, tnrange\n",
        "from tqdm.auto import tqdm\n",
        "from overrides import overrides\n",
        "from allennlp.data import Instance\n",
        "from allennlp.data.dataset_readers import DatasetReader\n",
        "from allennlp.common.file_utils import cached_path\n",
        "from allennlp.data.tokenizers import Tokenizer, WordTokenizer\n",
        "from allennlp.data.tokenizers.word_splitter import JustSpacesWordSplitter\n",
        "from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer\n",
        "from allennlp.data.tokenizers import Token\n",
        "from allennlp.data.vocabulary import Vocabulary\n",
        "from allennlp.data.fields import LabelField, TextField, Field, SequenceLabelField\n",
        "from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder\n",
        "from allennlp.modules.token_embedders import Embedding\n",
        "from allennlp.modules.seq2seq_encoders import Seq2SeqEncoder, PytorchSeq2SeqWrapper\n",
        "from allennlp.modules.seq2vec_encoders import PytorchSeq2VecWrapper\n",
        "from allennlp.modules.seq2vec_encoders import Seq2VecEncoder\n",
        "from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits\n",
        "from allennlp.data.iterators import BucketIterator\n",
        "\n",
        "from allennlp.training.trainer import Trainer\n",
        "\n",
        "from allennlp.predictors import SentenceTaggerPredictor\n",
        "from allennlp.data.token_indexers.elmo_indexer import ELMoTokenCharactersIndexer\n",
        "from allennlp.modules.token_embedders import ElmoTokenEmbedder\n",
        "\n",
        "from allennlp.nn.util import get_text_field_mask\n",
        "from allennlp.training.metrics import CategoricalAccuracy, F1Measure\n",
        "from allennlp.models import Model\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)\n",
        "\n",
        "logger = logging.getLogger(__name__) # pylint: disable=invalid-name\n",
        "EMBEDDING_DIM = 128\n",
        "HIDDEN_DIM = 60 #128\n",
        "MAX_LEN = 70\n",
        "dropout = 0.25\n",
        "lstm_layers = 2\n",
        "\n",
        "import io, random, os\n",
        "import tarfile\n",
        "import urllib.request\n",
        "import time\n",
        "\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\"\n",
        "\n",
        "tqdm.pandas(desc='Progress')\n",
        "\n",
        "# cross validation and metrics\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "SEED = 1033\n",
        "\n",
        "def seed_everything(seed=1033):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "seed_everything()\n",
        "\n",
        "data_dir = \"\""
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XSx0aNfjs6P5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "text_list = []\n",
        "\n",
        "def binaryStreamToList(source,dir_name):\n",
        "    res = []\n",
        "    \n",
        "    source = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'\n",
        "    tar = tarfile.open(fileobj=urllib.request.urlopen(source), mode=\"r|gz\")\n",
        "    for member in tar:\n",
        "        if member.name.endswith(\".txt\") and member.name.startswith(dir_name):\n",
        "            text_from_file = \"\"\n",
        "            content = tar.extractfile(member).read()\n",
        "            with io.BytesIO(content) as f:\n",
        "                for line in f:\n",
        "                    text_from_file += str(line).strip()\n",
        "                res.append(text_from_file)\n",
        "    return res\n",
        "\n",
        "def collect_df(source,dir_name,target):\n",
        "    text_list = binaryStreamToList(source,dir_name)\n",
        "    df = pd.DataFrame({'review_text': text_list,\n",
        "                  'target': target})\n",
        "    return df  \n",
        "  \n",
        "source = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7inxrklws6S1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "test_pos_tar = 'aclImdb/test/pos/'\n",
        "test_neg_tar = 'aclImdb/test/neg/'\n",
        "train_pos_tar = 'aclImdb/train/pos/'\n",
        "train_neg_tar = 'aclImdb/train/neg/'\n",
        "\n",
        "pos_train_df = collect_df(source,train_pos_tar,1)\n",
        "neg_train_df = collect_df(source,train_neg_tar,0)\n",
        "pos_test_df = collect_df(source,test_pos_tar,1)\n",
        "neg_test_df = collect_df(source,test_neg_tar,0)\n",
        "\n",
        "all_df = pd.concat([pos_train_df ,neg_train_df,pos_test_df ,neg_test_df]).sample(frac=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vAOzCK96s6WD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "outputId": "41b41006-7306-4e25-f27d-3589875ae864"
      },
      "cell_type": "code",
      "source": [
        "train_df, test_df = train_test_split(all_df, test_size=0.2)\n",
        "\n",
        "#check for duplicates\n",
        "duplicateRowsDF = all_df[all_df.duplicated()]\n",
        "print(len(train_df[train_df['target']==1]))\n",
        "print(len(train_df[train_df['target']==0]))\n",
        "print(len(test_df[test_df['target']==1]))\n",
        "print(len(test_df[test_df['target']==0]))\n",
        "\n",
        "train_df.to_csv(\"train.csv\")\n",
        "test_df.to_csv(\"test.csv\")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "20044\n",
            "19956\n",
            "4956\n",
            "5044\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "gYI7eCaXs6b-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def known_contractions(embed):\n",
        "    known = []\n",
        "    for contract in contraction_mapping:\n",
        "        if contract in embed:\n",
        "            known.append(contract)\n",
        "    return known\n",
        "def clean_contractions(text, mapping):\n",
        "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
        "    for s in specials:\n",
        "        text = text.replace(s, \"'\")\n",
        "    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n",
        "    return text\n",
        "def correct_spelling(x, dic):\n",
        "    for word in dic.keys():\n",
        "        x = x.replace(word, dic[word])\n",
        "    return x\n",
        "def unknown_punct(embed, punct):\n",
        "    unknown = ''\n",
        "    for p in punct:\n",
        "        if p not in embed:\n",
        "            unknown += p\n",
        "            unknown += ' '\n",
        "    return unknown\n",
        "\n",
        "def clean_special_chars(text, punct, mapping):\n",
        "    for p in mapping:\n",
        "        text = text.replace(p, mapping[p])\n",
        "    \n",
        "    for p in punct:\n",
        "        text = text.replace(p, f' {p} ')\n",
        "    \n",
        "    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n",
        "    for s in specials:\n",
        "        text = text.replace(s, specials[s])\n",
        "    \n",
        "    return text\n",
        "def add_lower(embedding, vocab):\n",
        "    count = 0\n",
        "    for word in vocab:\n",
        "        if word in embedding and word.lower() not in embedding:  \n",
        "            embedding[word.lower()] = embedding[word]\n",
        "            count += 1\n",
        "    print(f\"Added {count} words to embedding\")    \n",
        "\n",
        "puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
        " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
        " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
        " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
        " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n",
        "\n",
        "def clean_text(x):\n",
        "    x = str(x)\n",
        "    for punct in puncts:\n",
        "        x = x.replace(punct, f' {punct} ')\n",
        "    return x\n",
        "\n",
        "def clean_numbers(x):\n",
        "    x = re.sub('[0-9]{5,}', '#####', x)\n",
        "    x = re.sub('[0-9]{4}', '####', x)\n",
        "    x = re.sub('[0-9]{3}', '###', x)\n",
        "    x = re.sub('[0-9]{2}', '##', x)\n",
        "    return x\n",
        "\n",
        "mispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n",
        "\n",
        "def _get_mispell(mispell_dict):\n",
        "    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n",
        "    return mispell_dict, mispell_re\n",
        "\n",
        "mispellings, mispellings_re = _get_mispell(mispell_dict)\n",
        "def replace_typical_misspell(text):\n",
        "    def replace(match):\n",
        "        return mispellings[match.group(0)]\n",
        "    return mispellings_re.sub(replace, text)\n",
        "  \n",
        "def preprocess_text(text):\n",
        "    return replace_typical_misspell(clean_numbers(clean_text(text.strip().lower())))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wLgPxcaas6e9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#@DatasetReader.register(\"imdb_sentiment\")\n",
        "class ImdbSentimentDatasetReader(DatasetReader):\n",
        "    def __init__(self,\n",
        "                 lazy: bool = False,\n",
        "                 tokenizer: Tokenizer = None,\n",
        "                 token_indexers: Dict[str, TokenIndexer] = None) -> None:\n",
        "        super().__init__(lazy)\n",
        "        self._tokenizer = tokenizer or WordTokenizer(JustSpacesWordSplitter())\n",
        "        self._token_indexers = token_indexers or {\"tokens\": SingleIdTokenIndexer()}\n",
        "\n",
        "    @overrides\n",
        "    def _read(self, file_path):\n",
        "        logger.info(\"Reading instances from lines in file at: %s\", file_path)\n",
        "        with open(cached_path(file_path), \"r\") as data_file:\n",
        "            csv_in = csv.reader(data_file, delimiter=',')\n",
        "            next(csv_in)\n",
        "            for row in csv_in:\n",
        "                if len(row) == 3:\n",
        "                    review_text = preprocess_text(row[1])\n",
        "                    labels_list.append(row[3])\n",
        "                    yield self.text_to_instance(text=review_text, label=row[2])\n",
        "    @overrides\n",
        "    def text_to_instance(self,  # type: ignore\n",
        "                         text: str,\n",
        "                         label: str = None) -> Instance:\n",
        "        # pylint: disable=arguments-differ\n",
        "        fields: Dict[str, Field] = {}\n",
        "        tokenized_text = self._tokenizer.tokenize(text)\n",
        "        #torch.tensor(tokenized_text, dtype=torch.long).cuda()\n",
        "        fields[\"tokens\"] = TextField(tokenized_text[:MAX_LEN], self._token_indexers)\n",
        "        if label is not None:\n",
        "            fields['label'] = LabelField(label)\n",
        "        return Instance(fields)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "T4vFzkSMs6hu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256
        },
        "outputId": "b5f59fe5-8f2f-493d-82ea-c6636cd9d0f5"
      },
      "cell_type": "code",
      "source": [
        "labels_list = []\n",
        "\n",
        "elmo_token_indexer = ELMoTokenCharactersIndexer()\n",
        "reader = ImdbSentimentDatasetReader(\n",
        "    token_indexers={'tokens': elmo_token_indexer})\n",
        "\n",
        "train_dataset = reader.read(data_dir+\"train.csv\")\n",
        "dev_dataset = reader.read(data_dir+\"test.csv\")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0it [00:00, ?it/s]\u001b[A\n",
            "74it [00:00, 731.27it/s]\u001b[A\n",
            "158it [00:00, 759.63it/s]\u001b[A\n",
            "231it [00:00, 747.91it/s]\u001b[A\n",
            "310it [00:00, 759.87it/s]\u001b[A\n",
            "394it [00:00, 779.39it/s]\u001b[A\n",
            "485it [00:00, 813.77it/s]\u001b[A\n",
            "569it [00:00, 818.55it/s]\u001b[A\n",
            "655it [00:00, 827.59it/s]\u001b[A\n",
            "738it [00:00, 826.55it/s]\u001b[A\n",
            "40000it [00:52, 755.70it/s]\n",
            "10000it [00:12, 791.00it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "L-XB5O2os6ke",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "e73d059c-1eef-4d18-ceb5-7e6623b898a9"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "# Initialize the ELMo-based token embedder using a pre-trained file.\n",
        "# This takes a while if you run this script for the first time\n",
        "\n",
        "# Original\n",
        "# options_file = \"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json\"\n",
        "# weight_file = \"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5\"\n",
        "\n",
        "# Medium\n",
        "# options_file = \"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_options.json\"\n",
        "# weight_file = \"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x2048_256_2048cnn_1xhighway/elmo_2x2048_256_2048cnn_1xhighway_weights.hdf5\"\n",
        "\n",
        "# Use the 'Small' pre-trained model\n",
        "options_file = ('https://s3-us-west-2.amazonaws.com/allennlp/models/elmo'\n",
        "                '/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json')\n",
        "weight_file = ('https://s3-us-west-2.amazonaws.com/allennlp/models/elmo'\n",
        "               '/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5')\n",
        "\n",
        "elmo_embedder = ElmoTokenEmbedder(options_file, weight_file,2, dropout=0.1)\n",
        "\n",
        "vocab = Vocabulary.from_instances(train_dataset + dev_dataset,\n",
        "                                  min_count={'tokens': 3})\n",
        "\n",
        "# Pass in the ElmoTokenEmbedder instance instead\n",
        "word_embeddings = BasicTextFieldEmbedder({\"tokens\": elmo_embedder.to(device)})"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 336/336 [00:00<00:00, 131943.28B/s]\n",
            "100%|██████████| 54402456/54402456 [00:02<00:00, 19837492.39B/s]\n",
            "100%|██████████| 50000/50000 [00:00<00:00, 55727.30it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "C6osAUmrs6nM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "class LstmTwoClassifier(Model):\n",
        "    def __init__(self,\n",
        "                 word_embeddings: TextFieldEmbedder,\n",
        "                 encoder: Seq2VecEncoder,\n",
        "                 vocab: Vocabulary) -> None:\n",
        "        super().__init__(vocab)\n",
        "        # We need the embeddings to convert word IDs to their vector representations\n",
        "        self.word_embeddings = word_embeddings\n",
        "\n",
        "\n",
        "        self.encoder = encoder\n",
        "\n",
        "\n",
        "        self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(),\n",
        "                                          out_features=2)\n",
        "        self.accuracy = CategoricalAccuracy()\n",
        "        #self.projection = nn.Linear(self.encoder.get_output_dim(), out_sz)\n",
        "        self.out_act = torch.nn.Sigmoid()\n",
        "        \n",
        "\n",
        "        #self.loss_function = torch.nn.BCELoss()   #BCEWithLogitsLoss()   #CrossEntropyLoss()\n",
        "        self.loss = torch.nn.CrossEntropyLoss()  #torch.nn.BCEWithLogitsLoss()\n",
        "        \n",
        "\n",
        "    def forward(self,\n",
        "                tokens: Dict[str, torch.tensor],\n",
        "                label: torch.tensor = None) -> torch.tensor:\n",
        "\n",
        "        mask = get_text_field_mask(tokens)\n",
        "\n",
        "        # Forward pass\n",
        "        embeddings = self.word_embeddings(tokens)\n",
        "        encoder_out = self.encoder(embeddings, mask)\n",
        "        linear_out = self.hidden2tag(encoder_out)\n",
        "        \n",
        "        logits = self.out_act(linear_out)\n",
        "        \n",
        "        output = {\"logits\": logits}\n",
        "        if label is not None:\n",
        "            #y = torch.tensor(label.reshape(-1, 1), dtype=torch.float).cuda()\n",
        "            self.accuracy(logits, label)\n",
        "            #output[\"loss\"] = self.loss_function(logits, label)\n",
        "            output[\"loss\"] = self.loss(logits, label)\n",
        "            \n",
        "            \n",
        "        return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ifnFP-yUs6pt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "elmo_embedding_dim = 256\n",
        "lstm = PytorchSeq2VecWrapper(\n",
        "    torch.nn.LSTM(elmo_embedding_dim, HIDDEN_DIM, bidirectional=True,num_layers=lstm_layers, \n",
        "                            dropout = dropout, batch_first=True))\n",
        "\n",
        "#model = LstmBinaryClassifier(word_embeddings, lstm, vocab).cuda()\n",
        "\n",
        "model = LstmTwoClassifier(word_embeddings, lstm, vocab).cuda()\n",
        "\n",
        "optimizer = optim.Adam(model.parameters())\n",
        "\n",
        "loss_function = torch.nn.BCEWithLogitsLoss()\n",
        "\n",
        "batch_size = 128\n",
        "iterator = BucketIterator(batch_size=batch_size, sorting_keys=[(\"tokens\", \"num_tokens\")])\n",
        "\n",
        "#train_iter = torchtext.data.BucketIterator(dataset=train_dataset,batch_size=batch_size, sort_key=[(\"tokens\", \"num_tokens\")])\n",
        "#val_iter = torchtext.data.BucketIterator(dataset=dev_dataset,batch_size=batch_size, sort_key=[(\"tokens\", \"num_tokens\")])\n",
        "\n",
        "\n",
        "iterator.index_with(vocab)\n",
        "#train_iterator.index_with(vocab)\n",
        "#val_iterator.index_with(vocab)\n",
        "\n",
        "\n",
        "def save(m, info):\n",
        "    torch.save(info, data_dir+'best_model.info')\n",
        "    torch.save(m, data_dir+'best_model.m')\n",
        "    \n",
        "def load():\n",
        "    m = torch.load(data_dir+'best_model.m')\n",
        "    info = torch.load(data_dir+'best_model.info')\n",
        "    return m, info\n",
        "\n",
        "trainer = Trainer(model=model,\n",
        "                  optimizer=optimizer,\n",
        "                  iterator=iterator,\n",
        "                  train_dataset=train_dataset,\n",
        "                  validation_dataset=dev_dataset,\n",
        "                  patience=2,\n",
        "                  cuda_device=0,\n",
        "                  num_epochs=5)\n",
        "\n",
        "#summary(model, input_size=(1, 28, 28))\n",
        "#training(model=model, epoch=1, eval_every=500,\n",
        "#         loss_func=loss_function, optimizer=optimizer, train_iter=train_iter,\n",
        "#        val_iter=val_iter, warmup_epoch=0, early_stop=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HbH-Ie9qs6sO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        },
        "outputId": "019d0f1e-f3f9-4828-c96c-2f5529c29a63"
      },
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loss: 0.6354 ||: 100%|██████████| 313/313 [04:21<00:00,  1.38it/s]\n",
            "loss: 0.6051 ||: 100%|██████████| 79/79 [01:00<00:00,  1.90it/s]\n",
            "loss: 0.5848 ||: 100%|██████████| 313/313 [03:51<00:00,  1.36it/s]\n",
            "loss: 0.5767 ||: 100%|██████████| 79/79 [00:51<00:00,  1.90it/s]\n",
            "loss: 0.5544 ||: 100%|██████████| 313/313 [04:15<00:00,  1.35it/s]\n",
            "loss: 0.5471 ||: 100%|██████████| 79/79 [00:50<00:00,  1.91it/s]\n",
            "loss: 0.5311 ||: 100%|██████████| 313/313 [03:51<00:00,  1.34it/s]\n",
            "loss: 0.5293 ||: 100%|██████████| 79/79 [00:51<00:00,  1.91it/s]\n",
            "loss: 0.5225 ||: 100%|██████████| 313/313 [03:56<00:00,  1.36it/s]\n",
            "loss: 0.5219 ||: 100%|██████████| 79/79 [00:51<00:00,  1.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'best_epoch': 4,\n",
              " 'best_validation_loss': 0.5218760948392409,\n",
              " 'epoch': 4,\n",
              " 'peak_cpu_memory_MB': 10214.496,\n",
              " 'peak_gpu_0_memory_MB': 3646,\n",
              " 'training_cpu_memory_MB': 10214.496,\n",
              " 'training_duration': '00:24:44',\n",
              " 'training_epochs': 4,\n",
              " 'training_gpu_0_memory_MB': 3646,\n",
              " 'training_loss': 0.5224993031817122,\n",
              " 'training_start_epoch': 0,\n",
              " 'validation_loss': 0.5218760948392409}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "metadata": {
        "id": "I3aQicZPs6vA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "74f09a68-ed51-4785-e175-dfd167731d82"
      },
      "cell_type": "code",
      "source": [
        "model_file_path = data_dir+\"model_allen_imdb_twoclass.th\"\n",
        "vocab_file_path = data_dir+\"vocabulary_allennlp_imdb_twoclass\"\n",
        "\n",
        "with open(model_file_path, 'wb') as f:\n",
        "    torch.save(model.state_dict(), f)\n",
        "\n",
        "vocab.save_to_files(vocab_file_path)\n",
        "\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:vocabulary serialization directory vocabulary_allennlp_imdb_twoclass is not empty\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "w4rRk6d44zKS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "25425c14-839e-45d9-f7f4-736bc8c2d495"
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'glove.6B.300d (1).txt'         test.csv\t\t\t    x_train.npy\n",
            " glove.6B.300d.txt\t        train.csv\t\t\t    y_test.npy\n",
            " glove.6B.zip\t\t        vocabulary_allennlp_imdb_twoclass   y_train.npy\n",
            " model_allen_imdb_twoclass.th   word_index.npy\n",
            " sample_data\t\t        x_test.npy\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Oc4wHgDD5dx8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c7526e32-61a2-4f47-d25e-116824b6b7c3"
      },
      "cell_type": "code",
      "source": [
        "vocab"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Vocabulary with namespaces:  labels, Size: 2 || Non Padded Namespaces: {'*tags', '*labels'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "metadata": {
        "id": "laWMC5Hms6xs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "from allennlp.common import JsonDict\n",
        "from allennlp.data import DatasetReader, Instance\n",
        "from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter\n",
        "from allennlp.predictors import Predictor\n",
        "from overrides import overrides\n",
        "from typing import List\n",
        "\n",
        "#@Predictor.register(\"sentence_classifier_predictor\")\n",
        "class SentenceClassifierPredictor(Predictor):\n",
        "    def __init__(self, model: Model, dataset_reader: DatasetReader) -> None:\n",
        "        super().__init__(model, dataset_reader)\n",
        "        self._tokenizer = SpacyWordSplitter(language='en_core_web_sm', pos_tags=True)\n",
        "\n",
        "    def predict(self, sentence: str) -> JsonDict:\n",
        "        return self.predict_json({\"sentence\" : sentence})\n",
        "\n",
        "    @overrides\n",
        "    def _json_to_instance(self, json_dict: JsonDict) -> Instance:\n",
        "        sentence = json_dict[\"sentence\"]\n",
        "        tokens = self._tokenizer.split_words(sentence)\n",
        "        #return self._dataset_reader.text_to_instance([str(t) for t in tokens])\n",
        "        return self._dataset_reader.text_to_instance(sentence)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "49wYeOqZs60e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        },
        "outputId": "aba5086c-e740-43eb-d642-2f3e62937fa3"
      },
      "cell_type": "code",
      "source": [
        "tests = ['this movie sucks','I love this movie','wow this is the worst film ever','I hated it, really. Do not watch this','it is pretty good, I would see it again']\n",
        "\n",
        "for test in tests:\n",
        "  predictor = SentenceClassifierPredictor(model, dataset_reader=reader)\n",
        "  logits = predictor.predict(test)['logits']\n",
        "  label_id = np.argmax(logits)\n",
        "\n",
        "  print(model.vocab.get_token_from_index(label_id, 'labels'))\n",
        "  \n",
        "  print(logits)\n",
        "  "
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "[0.9339547753334045, 0.06673074513673782]\n",
            "1\n",
            "[0.010839399881660938, 0.9916225075721741]\n",
            "0\n",
            "[0.9991610050201416, 0.0006861151778139174]\n",
            "0\n",
            "[0.6092225909233093, 0.40380367636680603]\n",
            "1\n",
            "[0.06797058880329132, 0.9387489557266235]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "k6xlP0Lus63L",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8HqHv9h7s65n",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bIThu1_6s69B",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}