nlp/golve_vectors_with_keras_deep_nets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "golve vectors with keras deep nets",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experments/blob/master/nlp/golve_vectors_with_keras_deep_nets.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QaOlHMtotkhz",
        "colab_type": "code",
        "outputId": "92d4e88d-e653-49d8-ff0f-a6b353b97b43",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "!wget https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-11-08 06:39:02--  https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.217.32.230\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.217.32.230|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1647046227 (1.5G) [application/x-gzip]\n",
            "Saving to: ‘GoogleNews-vectors-negative300.bin.gz’\n",
            "\n",
            "GoogleNews-vectors- 100%[===================>]   1.53G  33.3MB/s    in 46s     \n",
            "\n",
            "2019-11-08 06:39:48 (34.4 MB/s) - ‘GoogleNews-vectors-negative300.bin.gz’ saved [1647046227/1647046227]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3Heh0sxt5wI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!gzip -df GoogleNews-vectors-negative300.bin.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExO9H_vNzOMm",
        "colab_type": "code",
        "outputId": "f5cf8cbd-02ca-48af-dd9d-2156e6f107c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "!python -m spacy download en_core_web_sm"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: en_core_web_sm==2.1.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "teZ92umcDPXl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://nlpforhackers.io/keras-intro/\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import spacy \n",
        "nlp = spacy.load(\"en\")\n",
        "\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "\n",
        "\n",
        "import matplotlib.pyplot as plt;\n",
        "from itertools import cycle;\n",
        "import matplotlib.pyplot as plt;\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from collections import defaultdict\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "woFJ_aIvTy5d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(comment, lowercase=True, remove_stopwords=True):\n",
        "    if lowercase:\n",
        "        comment = comment.lower()\n",
        "    lines = comment.splitlines()\n",
        "    lines = [x.strip(' ') for x in lines]\n",
        "    lines = [x.replace('\"', '') for x in lines]\n",
        "    lines = [x.replace('\\\\\"', '') for x in lines]\n",
        "    lines = [x.replace(u'\\xa0', u'') for x in lines]\n",
        "    comment = \" \".join(lines)\n",
        "    doc = nlp(comment)\n",
        "\n",
        "    # for token in doc:\n",
        "    #   print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n",
        "    #     token.shape_, token.is_alpha, token.is_stop)\n",
        "\n",
        "    words = [token for token in doc if token.is_stop !=\n",
        "             True and token.is_punct != True]\n",
        "    # return \" \".join(words)\n",
        "    lemmatized = list()\n",
        "    for word in words:\n",
        "        lemma = word.lemma_.strip()\n",
        "        if lemma:\n",
        "            lemmatized.append(lemma)\n",
        "    return lemmatized"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uqloCGmfT0pb",
        "colab_type": "code",
        "outputId": "6e490441-d29a-403e-deb5-45cb4583c228",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "import os\n",
        "\n",
        "def writetofile(dir, filename, data):\n",
        "    if not os.path.exists(dir):\n",
        "        os.makedirs(dir)\n",
        "    f = os.path.join(dir,str(filename))\n",
        "\n",
        "    with open(f, 'wb') as the_file:\n",
        "      the_file.write(data)\n",
        "\n",
        "\n",
        "categories = [ 'comp.graphics','comp.os.ms-windows.misc','comp.sys.ibm.pc.hardware','comp.sys.mac.hardware','comp.windows.x']\n",
        "news = fetch_20newsgroups(subset=\"train\", categories=categories)\n",
        "\n",
        "\n",
        "clean_data  = []\n",
        "\n",
        "# print(news.keys())\n",
        "\n",
        "# print(news[\"filenames\"][:10])\n",
        "\n",
        "# print(len(news[\"data\"][:1000]))\n",
        "\n",
        "# print(news[\"target_names\"][:50])\n",
        "# print(news[\"target\"][:10])\n",
        "\n",
        "\n",
        "max_limit = 4000\n",
        "\n",
        "targets = news[\"target\"][:max_limit]\n",
        "filenames = news[\"filenames\"][:max_limit]\n",
        "news = news[\"data\"][:max_limit]\n",
        "\n",
        "dir = \"news_group_cleaned\"\n",
        "\n",
        "total_words = 0\n",
        "\n",
        "print(\"cleaning data\")\n",
        "for i, row in enumerate(news):\n",
        "  # filename = filenames[i]\n",
        "  # filename = filename[(filename.rfind('/'))+1:]\n",
        "  # if os.path.exists(filename):\n",
        "  #   with open(os.path.join(dir, filename), 'r') as content_file:\n",
        "  #     data = content_file.read()\n",
        "  #     cleaned = data.split(\" \")\n",
        "  #     total_words += len(cleaned)\n",
        "  # else:\n",
        "  cleaned = normalize(row)\n",
        "  cleaned = \" \".join(cleaned)\n",
        "    # writetofile(dir, filename, cleaned.encode(\"utf-8\"))\n",
        "  total_words += len(cleaned)\n",
        "  clean_data.append(cleaned)\n",
        "\n",
        "print(\"data cleaned\")\n",
        "\n",
        "print(clean_data[1000])\n",
        "\n",
        "print('total words' , total_words)\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading 20news dataset. This may take a few minutes.\n",
            "Downloading dataset from https://ndownloader.figshare.com/files/5975967 (14 MB)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "cleaning data\n",
            "data cleaned\n",
            "rcaldrn@med.miami.edu richard calderon subject move icon nntp post host epistat.med.miami.edu organization university miami medical school line 17 kmembry@viamar.uucp kirk membry write > remember read program window icon run away > mouse move near know > program ftp location probably cica remember program look call icofrite cica see ago richard calderon rcaldrn@epi.med.miami.edu university miami school medicine information system compute 1029 nw 15 st miami florida 33136\n",
            "total words 3801696\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8_9d79XvpnB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "535811d5-9112-49c2-9771-725330ed66a2"
      },
      "source": [
        "import gensim\n",
        "model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True) \n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MwJ_rz-wS0Pl",
        "colab_type": "code",
        "outputId": "159c8f94-f521-4e58-d134-0f5314ae572d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133
        }
      },
      "source": [
        "#prepare text\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "MAX_SEQUENCE_LENGTH = 1000\n",
        "VALIDATION_SPLIT = 0.2\n",
        "\n",
        "\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(clean_data)\n",
        "sequences = tokenizer.texts_to_sequences(clean_data)\n",
        "\n",
        "# sequences2 = tokenizer.text_to_word_sequence(clean_data) \n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "\n",
        "\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n",
        "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "# print(clean_data[0])\n",
        "# print(data[0])\n",
        "\n",
        "# print(word_index)\n",
        "\n",
        "labels = to_categorical(np.asarray(targets))\n",
        "print('Shape of data tensor:', data.shape)\n",
        "print('Shape of label tensor:', labels.shape)\n",
        "\n",
        "# split the data into a training set and a validation set\n",
        "indices = np.arange(data.shape[0])\n",
        "np.random.shuffle(indices)\n",
        "data = data[indices]\n",
        "labels = labels[indices]\n",
        "nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
        "\n",
        "x_train = data[:-nb_validation_samples]\n",
        "y_train = labels[:-nb_validation_samples]\n",
        "x_val = data[-nb_validation_samples:]\n",
        "y_val = labels[-nb_validation_samples:]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Found 64405 unique tokens.\n",
            "Shape of data tensor: (2936, 1000)\n",
            "Shape of label tensor: (2936, 5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KEHleYqG9Uz",
        "colab_type": "code",
        "outputId": "814f4b89-e563-4f85-80ac-eb52cc8fda09",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "EMBEDDINGS_LEN = 300\n",
        "\n",
        "\n",
        "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDINGS_LEN))\n",
        "for word, idx in word_index.items():\n",
        "    try:\n",
        "        embedding = model.wv[word]\n",
        "        embedding_matrix[idx] = embedding\n",
        "    except:\n",
        "      pass"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EciNXLQ_j4yU",
        "colab_type": "code",
        "outputId": "9631a46b-942d-4a06-d3e7-86c374f428d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "\n",
        "from keras.layers import Dense\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Embedding, Conv1D, MaxPooling1D, Flatten\n",
        "\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Embedding(len(word_index) + 1,\n",
        "                            EMBEDDINGS_LEN,\n",
        "                            weights=[embedding_matrix],\n",
        "                            input_length=MAX_SEQUENCE_LENGTH,\n",
        "                            trainable=False))\n",
        "model.add(Conv1D(128, 5, activation='relu'))\n",
        "model.add(MaxPooling1D(5))\n",
        "model.add(Conv1D(128, 5, activation='relu'))\n",
        "model.add(MaxPooling1D(5))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(units=128, activation='relu'))\n",
        "model.add(Dense(units=len(categories), activation='softmax'))\n",
        "\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()\n",
        "\n",
        "\n",
        "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
        "          epochs=12, batch_size=128,shuffle=True)\n",
        "\n",
        "# scores = model.evaluate(X_test, Y_test, verbose=1)\n",
        "\n",
        "# print(\"Accuracy: \", scores[1])\n",
        "# print(\"Loss: \", scores[0])\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 1000, 300)         19321800  \n",
            "_________________________________________________________________\n",
            "conv1d_1 (Conv1D)            (None, 996, 128)          192128    \n",
            "_________________________________________________________________\n",
            "max_pooling1d_1 (MaxPooling1 (None, 199, 128)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_2 (Conv1D)            (None, 195, 128)          82048     \n",
            "_________________________________________________________________\n",
            "max_pooling1d_2 (MaxPooling1 (None, 39, 128)           0         \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 4992)              0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 128)               639104    \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 5)                 645       \n",
            "=================================================================\n",
            "Total params: 20,235,725\n",
            "Trainable params: 913,925\n",
            "Non-trainable params: 19,321,800\n",
            "_________________________________________________________________\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "Train on 2349 samples, validate on 587 samples\n",
            "Epoch 1/12\n",
            "2349/2349 [==============================] - 45s 19ms/step - loss: 1.6092 - acc: 0.2256 - val_loss: 1.5918 - val_acc: 0.2419\n",
            "Epoch 2/12\n",
            "2349/2349 [==============================] - 45s 19ms/step - loss: 1.4636 - acc: 0.3580 - val_loss: 1.2923 - val_acc: 0.3901\n",
            "Epoch 3/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 1.0515 - acc: 0.5819 - val_loss: 1.0570 - val_acc: 0.5758\n",
            "Epoch 4/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.7273 - acc: 0.7160 - val_loss: 0.9765 - val_acc: 0.6133\n",
            "Epoch 5/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.5190 - acc: 0.8025 - val_loss: 0.8024 - val_acc: 0.7138\n",
            "Epoch 6/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.3200 - acc: 0.9110 - val_loss: 0.8002 - val_acc: 0.7308\n",
            "Epoch 7/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.2061 - acc: 0.9498 - val_loss: 0.7693 - val_acc: 0.7513\n",
            "Epoch 8/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.1528 - acc: 0.9591 - val_loss: 0.9042 - val_acc: 0.7223\n",
            "Epoch 9/12\n",
            "2349/2349 [==============================] - 43s 18ms/step - loss: 0.0906 - acc: 0.9864 - val_loss: 0.8368 - val_acc: 0.7496\n",
            "Epoch 10/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.0569 - acc: 0.9932 - val_loss: 0.8828 - val_acc: 0.7479\n",
            "Epoch 11/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.0338 - acc: 0.9966 - val_loss: 0.8679 - val_acc: 0.7751\n",
            "Epoch 12/12\n",
            "2349/2349 [==============================] - 44s 19ms/step - loss: 0.0206 - acc: 0.9987 - val_loss: 0.9223 - val_acc: 0.7581\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fc28cf7b438>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZHXtWsarE6o",
        "colab_type": "code",
        "outputId": "8dd4e093-9730-4e74-ea69-742765e496a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 921
        }
      },
      "source": [
        "from keras.layers import Dense\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Embedding, Conv1D, MaxPooling1D, Flatten\n",
        "from keras.layers import LSTM\n",
        "\n",
        "\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Embedding(len(word_index) + 1,\n",
        "                            EMBEDDINGS_LEN,\n",
        "                            weights=[embedding_matrix],\n",
        "                            input_length=MAX_SEQUENCE_LENGTH,\n",
        "                            trainable=False))\n",
        "model.add(Conv1D(128, 5, activation='relu'))\n",
        "model.add(MaxPooling1D(5))\n",
        "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
        "model.add(Dense(units=128, activation='relu'))\n",
        "model.add(Dense(units=len(categories), activation='softmax'))\n",
        "\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()\n",
        "\n",
        "\n",
        "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
        "          epochs=12, batch_size=32,shuffle=True)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 1000, 300)         19321800  \n",
            "_________________________________________________________________\n",
            "conv1d_3 (Conv1D)            (None, 996, 128)          192128    \n",
            "_________________________________________________________________\n",
            "max_pooling1d_3 (MaxPooling1 (None, 199, 128)          0         \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 128)               131584    \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 128)               16512     \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 5)                 645       \n",
            "=================================================================\n",
            "Total params: 19,662,669\n",
            "Trainable params: 340,869\n",
            "Non-trainable params: 19,321,800\n",
            "_________________________________________________________________\n",
            "Train on 2349 samples, validate on 587 samples\n",
            "Epoch 1/12\n",
            "2349/2349 [==============================] - 68s 29ms/step - loss: 1.4523 - acc: 0.3333 - val_loss: 1.3234 - val_acc: 0.3424\n",
            "Epoch 2/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 1.0036 - acc: 0.5871 - val_loss: 0.9429 - val_acc: 0.6559\n",
            "Epoch 3/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.7149 - acc: 0.7314 - val_loss: 0.8079 - val_acc: 0.7019\n",
            "Epoch 4/12\n",
            "2349/2349 [==============================] - 65s 28ms/step - loss: 0.5120 - acc: 0.8157 - val_loss: 0.8662 - val_acc: 0.7411\n",
            "Epoch 5/12\n",
            "2349/2349 [==============================] - 65s 28ms/step - loss: 0.4459 - acc: 0.8489 - val_loss: 0.6694 - val_acc: 0.7632\n",
            "Epoch 6/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.3328 - acc: 0.8804 - val_loss: 0.5521 - val_acc: 0.8007\n",
            "Epoch 7/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.2402 - acc: 0.9200 - val_loss: 0.5403 - val_acc: 0.8092\n",
            "Epoch 8/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.1737 - acc: 0.9451 - val_loss: 0.5517 - val_acc: 0.8330\n",
            "Epoch 9/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.1460 - acc: 0.9527 - val_loss: 0.5273 - val_acc: 0.8177\n",
            "Epoch 10/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.1271 - acc: 0.9591 - val_loss: 0.6478 - val_acc: 0.7973\n",
            "Epoch 11/12\n",
            "2349/2349 [==============================] - 65s 28ms/step - loss: 0.1037 - acc: 0.9715 - val_loss: 0.6272 - val_acc: 0.8143\n",
            "Epoch 12/12\n",
            "2349/2349 [==============================] - 66s 28ms/step - loss: 0.1381 - acc: 0.9570 - val_loss: 0.6741 - val_acc: 0.8143\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fc2d6627240>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    }
  ]
}