Market_Price_Prediction_with_Market_Sentiment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Market Price Prediction with Market Sentiment.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "lYuohdJHmpIp",
        "colab_type": "code",
        "outputId": "4bf882e3-2b46-4170-ec3f-4f31402a2024",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import io\n",
        "import re\n",
        "import sys\n",
        "!{sys.executable} -m pip install gensim\n",
        "!{sys.executable} -m pip install nltk\n",
        "import gensim\n",
        "import nltk\n",
        "nltk.download('punkt') #nltk Dependency\n",
        "from gensim.models.doc2vec import Doc2Vec,TaggedDocument\n",
        "from nltk.tokenize import word_tokenize\n",
        "from datetime import datetime,date,timedelta\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gensim\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/a4/d10c0acc8528d838cda5eede0ee9c784caa598dbf40bd0911ff8d067a7eb/gensim-3.6.0-cp36-cp36m-manylinux1_x86_64.whl (23.6MB)\n",
            "\u001b[K    100% |████████████████████████████████| 23.6MB 1.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.11.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.14.6)\n",
            "Collecting smart-open>=1.2.1 (from gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/4b/1f/6f27e3682124de63ac97a0a5876da6186de6c19410feab66c1543afab055/smart_open-1.7.1.tar.gz\n",
            "Collecting boto>=2.32 (from smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/23/10/c0b78c27298029e4454a472a1919bde20cb182dab1662cec7f2ca1dcc523/boto-2.49.0-py2.py3-none-any.whl (1.4MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.4MB 16.1MB/s \n",
            "\u001b[?25hCollecting bz2file (from smart-open>=1.2.1->gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/61/39/122222b5e85cd41c391b68a99ee296584b2a2d1d233e7ee32b4532384f2d/bz2file-0.98.tar.gz\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.18.4)\n",
            "Collecting boto3 (from smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bf/a1/2fedb80d3eefe024580aaff3e81106058b6f99698295edfca51199162bd5/boto3-1.9.57-py2.py3-none-any.whl (128kB)\n",
            "\u001b[K    100% |████████████████████████████████| 133kB 28.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2018.10.15)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.6)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.22)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
            "Collecting s3transfer<0.2.0,>=0.1.10 (from boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/14/2a0004d487464d120c9fb85313a75cd3d71a7506955be458eebfe19a6b1d/s3transfer-0.1.13-py2.py3-none-any.whl (59kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 23.3MB/s \n",
            "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1 (from boto3->smart-open>=1.2.1->gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/b7/31/05c8d001f7f87f0f07289a5fc0fc3832e9a57f2dbd4d3b0fee70e0d51365/jmespath-0.9.3-py2.py3-none-any.whl\n",
            "Collecting botocore<1.13.0,>=1.12.57 (from boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f1/37/eb8f5a76e1cb16ecabb7c92f7504c37030c8b727d550021b2bb34dc2a082/botocore-1.12.57-py2.py3-none-any.whl (5.1MB)\n",
            "\u001b[K    100% |████████████████████████████████| 5.1MB 7.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.57->boto3->smart-open>=1.2.1->gensim) (2.5.3)\n",
            "Collecting docutils>=0.10 (from botocore<1.13.0,>=1.12.57->boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/36/fa/08e9e6e0e3cbd1d362c3bbee8d01d0aedb2155c4ac112b19ef3cae8eed8d/docutils-0.14-py3-none-any.whl (543kB)\n",
            "\u001b[K    100% |████████████████████████████████| 552kB 25.3MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: smart-open, bz2file\n",
            "  Running setup.py bdist_wheel for smart-open ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/23/00/44/e5b939f7a80c04e32297dbd6d96fa3065af89ecf57e2b5f89f\n",
            "  Running setup.py bdist_wheel for bz2file ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/81/75/d6/e1317bf09bf1af5a30befc2a007869fa6e1f516b8f7c591cb9\n",
            "Successfully built smart-open bz2file\n",
            "Installing collected packages: boto, bz2file, jmespath, docutils, botocore, s3transfer, boto3, smart-open, gensim\n",
            "Successfully installed boto-2.49.0 boto3-1.9.57 botocore-1.12.57 bz2file-0.98 docutils-0.14 gensim-3.6.0 jmespath-0.9.3 s3transfer-0.1.13 smart-open-1.7.1\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.11.0)\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "outputId": "f31d876a-b1ce-4e86-ec11-2eac96b836ea",
        "id": "EFARkTW1-J9v",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "gaaX4hxW9rri",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "newsDataframe = pd.read_csv('drive/My Drive/stockNewsDataset/newsdata.csv')\n",
        "marketDataframe = pd.read_csv('drive/My Drive/stockNewsDataset/marketdata.csv')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4HpNqWrVrRsq",
        "colab_type": "code",
        "outputId": "ced18241-e44f-4c8b-9d33-bce461b1ced4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 596
        }
      },
      "cell_type": "code",
      "source": [
        "newsDataframe.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>time</th>\n",
              "      <th>sourceTimestamp</th>\n",
              "      <th>firstCreated</th>\n",
              "      <th>sourceId</th>\n",
              "      <th>headline</th>\n",
              "      <th>urgency</th>\n",
              "      <th>takeSequence</th>\n",
              "      <th>provider</th>\n",
              "      <th>subjects</th>\n",
              "      <th>...</th>\n",
              "      <th>noveltyCount12H</th>\n",
              "      <th>noveltyCount24H</th>\n",
              "      <th>noveltyCount3D</th>\n",
              "      <th>noveltyCount5D</th>\n",
              "      <th>noveltyCount7D</th>\n",
              "      <th>volumeCounts12H</th>\n",
              "      <th>volumeCounts24H</th>\n",
              "      <th>volumeCounts3D</th>\n",
              "      <th>volumeCounts5D</th>\n",
              "      <th>volumeCounts7D</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>2007-01-01 04:29:32+00:00</td>\n",
              "      <td>2007-01-01 04:29:32+00:00</td>\n",
              "      <td>2007-01-01 04:29:32+00:00</td>\n",
              "      <td>e58c6279551b85cf</td>\n",
              "      <td>China's Daqing pumps 43.41 mln tonnes of oil i...</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>RTRS</td>\n",
              "      <td>{'ENR', 'ASIA', 'CN', 'NGS', 'EMRG', 'RTRS', '...</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2007-01-01 07:03:35+00:00</td>\n",
              "      <td>2007-01-01 07:03:34+00:00</td>\n",
              "      <td>2007-01-01 07:03:34+00:00</td>\n",
              "      <td>5a31c4327427f63f</td>\n",
              "      <td>FEATURE-In kidnapping, finesse works best</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>RTRS</td>\n",
              "      <td>{'FEA', 'CA', 'LATAM', 'MX', 'INS', 'ASIA', 'I...</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2007-01-01 11:29:56+00:00</td>\n",
              "      <td>2007-01-01 11:29:56+00:00</td>\n",
              "      <td>2007-01-01 11:29:56+00:00</td>\n",
              "      <td>1cefd27a40fabdfe</td>\n",
              "      <td>PRESS DIGEST - Wall Street Journal - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>RTRS</td>\n",
              "      <td>{'RET', 'ENR', 'ID', 'BG', 'US', 'PRESS', 'IQ'...</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>17.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>23768af19dc69992</td>\n",
              "      <td>PRESS DIGEST - New York Times - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>RTRS</td>\n",
              "      <td>{'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>15.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>2007-01-01 12:08:37+00:00</td>\n",
              "      <td>23768af19dc69992</td>\n",
              "      <td>PRESS DIGEST - New York Times - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>RTRS</td>\n",
              "      <td>{'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 36 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0                       time            sourceTimestamp  \\\n",
              "0           0  2007-01-01 04:29:32+00:00  2007-01-01 04:29:32+00:00   \n",
              "1           1  2007-01-01 07:03:35+00:00  2007-01-01 07:03:34+00:00   \n",
              "2           2  2007-01-01 11:29:56+00:00  2007-01-01 11:29:56+00:00   \n",
              "3           3  2007-01-01 12:08:37+00:00  2007-01-01 12:08:37+00:00   \n",
              "4           4  2007-01-01 12:08:37+00:00  2007-01-01 12:08:37+00:00   \n",
              "\n",
              "                firstCreated          sourceId  \\\n",
              "0  2007-01-01 04:29:32+00:00  e58c6279551b85cf   \n",
              "1  2007-01-01 07:03:34+00:00  5a31c4327427f63f   \n",
              "2  2007-01-01 11:29:56+00:00  1cefd27a40fabdfe   \n",
              "3  2007-01-01 12:08:37+00:00  23768af19dc69992   \n",
              "4  2007-01-01 12:08:37+00:00  23768af19dc69992   \n",
              "\n",
              "                                            headline  urgency  takeSequence  \\\n",
              "0  China's Daqing pumps 43.41 mln tonnes of oil i...        3             1   \n",
              "1          FEATURE-In kidnapping, finesse works best        3             1   \n",
              "2         PRESS DIGEST - Wall Street Journal - Jan 1        3             1   \n",
              "3              PRESS DIGEST - New York Times - Jan 1        3             1   \n",
              "4              PRESS DIGEST - New York Times - Jan 1        3             1   \n",
              "\n",
              "  provider                                           subjects       ...        \\\n",
              "0     RTRS  {'ENR', 'ASIA', 'CN', 'NGS', 'EMRG', 'RTRS', '...       ...         \n",
              "1     RTRS  {'FEA', 'CA', 'LATAM', 'MX', 'INS', 'ASIA', 'I...       ...         \n",
              "2     RTRS  {'RET', 'ENR', 'ID', 'BG', 'US', 'PRESS', 'IQ'...       ...         \n",
              "3     RTRS  {'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...       ...         \n",
              "4     RTRS  {'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...       ...         \n",
              "\n",
              "  noveltyCount12H  noveltyCount24H  noveltyCount3D noveltyCount5D  \\\n",
              "0             0.0              0.0             0.0            0.0   \n",
              "1             1.0              1.0             1.0            1.0   \n",
              "2             0.0              0.0             0.0            0.0   \n",
              "3             0.0              0.0             0.0            0.0   \n",
              "4             0.0              0.0             0.0            0.0   \n",
              "\n",
              "   noveltyCount7D  volumeCounts12H  volumeCounts24H volumeCounts3D  \\\n",
              "0             0.0              0.0              0.0            3.0   \n",
              "1             1.0              1.0              1.0            3.0   \n",
              "2             0.0              0.0              0.0            5.0   \n",
              "3             0.0              0.0              0.0            5.0   \n",
              "4             0.0              0.0              0.0            0.0   \n",
              "\n",
              "  volumeCounts5D  volumeCounts7D  \n",
              "0            6.0             7.0  \n",
              "1            3.0             3.0  \n",
              "2           11.0            17.0  \n",
              "3           13.0            15.0  \n",
              "4            0.0             0.0  \n",
              "\n",
              "[5 rows x 36 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "10qrD3-qrcyT",
        "colab_type": "code",
        "outputId": "43ba3a33-e7af-42cf-95ca-5e5e4b2b5179",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 341
        }
      },
      "cell_type": "code",
      "source": [
        "marketDataframe.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>time</th>\n",
              "      <th>assetCode</th>\n",
              "      <th>assetName</th>\n",
              "      <th>volume</th>\n",
              "      <th>close</th>\n",
              "      <th>open</th>\n",
              "      <th>returnsClosePrevRaw1</th>\n",
              "      <th>returnsOpenPrevRaw1</th>\n",
              "      <th>returnsClosePrevMktres1</th>\n",
              "      <th>returnsOpenPrevMktres1</th>\n",
              "      <th>returnsClosePrevRaw10</th>\n",
              "      <th>returnsOpenPrevRaw10</th>\n",
              "      <th>returnsClosePrevMktres10</th>\n",
              "      <th>returnsOpenPrevMktres10</th>\n",
              "      <th>returnsOpenNextMktres10</th>\n",
              "      <th>universe</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>2007-02-01 22:00:00+00:00</td>\n",
              "      <td>A.N</td>\n",
              "      <td>Agilent Technologies Inc</td>\n",
              "      <td>2606900.0</td>\n",
              "      <td>32.19</td>\n",
              "      <td>32.17</td>\n",
              "      <td>0.005938</td>\n",
              "      <td>0.005312</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.001860</td>\n",
              "      <td>0.000622</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.034672</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2007-02-01 22:00:00+00:00</td>\n",
              "      <td>AAI.N</td>\n",
              "      <td>AirTran Holdings Inc</td>\n",
              "      <td>2051600.0</td>\n",
              "      <td>11.12</td>\n",
              "      <td>11.08</td>\n",
              "      <td>0.004517</td>\n",
              "      <td>-0.007168</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.078708</td>\n",
              "      <td>-0.088066</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.027803</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2007-02-01 22:00:00+00:00</td>\n",
              "      <td>AAP.N</td>\n",
              "      <td>Advance Auto Parts Inc</td>\n",
              "      <td>1164800.0</td>\n",
              "      <td>37.51</td>\n",
              "      <td>37.99</td>\n",
              "      <td>-0.011594</td>\n",
              "      <td>0.025648</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.014332</td>\n",
              "      <td>0.045405</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.024433</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>2007-02-01 22:00:00+00:00</td>\n",
              "      <td>AAPL.O</td>\n",
              "      <td>Apple Inc</td>\n",
              "      <td>23747329.0</td>\n",
              "      <td>84.74</td>\n",
              "      <td>86.23</td>\n",
              "      <td>-0.011548</td>\n",
              "      <td>0.016324</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.048613</td>\n",
              "      <td>-0.037182</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.007425</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>2007-02-01 22:00:00+00:00</td>\n",
              "      <td>ABB.N</td>\n",
              "      <td>ABB Ltd</td>\n",
              "      <td>1208600.0</td>\n",
              "      <td>18.02</td>\n",
              "      <td>18.01</td>\n",
              "      <td>0.011791</td>\n",
              "      <td>0.025043</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.012929</td>\n",
              "      <td>0.020397</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>-0.017994</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0                       time assetCode                 assetName  \\\n",
              "0           0  2007-02-01 22:00:00+00:00       A.N  Agilent Technologies Inc   \n",
              "1           1  2007-02-01 22:00:00+00:00     AAI.N      AirTran Holdings Inc   \n",
              "2           2  2007-02-01 22:00:00+00:00     AAP.N    Advance Auto Parts Inc   \n",
              "3           3  2007-02-01 22:00:00+00:00    AAPL.O                 Apple Inc   \n",
              "4           4  2007-02-01 22:00:00+00:00     ABB.N                   ABB Ltd   \n",
              "\n",
              "       volume  close   open  returnsClosePrevRaw1  returnsOpenPrevRaw1  \\\n",
              "0   2606900.0  32.19  32.17              0.005938             0.005312   \n",
              "1   2051600.0  11.12  11.08              0.004517            -0.007168   \n",
              "2   1164800.0  37.51  37.99             -0.011594             0.025648   \n",
              "3  23747329.0  84.74  86.23             -0.011548             0.016324   \n",
              "4   1208600.0  18.02  18.01              0.011791             0.025043   \n",
              "\n",
              "   returnsClosePrevMktres1  returnsOpenPrevMktres1  returnsClosePrevRaw10  \\\n",
              "0                      NaN                     NaN              -0.001860   \n",
              "1                      NaN                     NaN              -0.078708   \n",
              "2                      NaN                     NaN               0.014332   \n",
              "3                      NaN                     NaN              -0.048613   \n",
              "4                      NaN                     NaN               0.012929   \n",
              "\n",
              "   returnsOpenPrevRaw10  returnsClosePrevMktres10  returnsOpenPrevMktres10  \\\n",
              "0              0.000622                       NaN                      NaN   \n",
              "1             -0.088066                       NaN                      NaN   \n",
              "2              0.045405                       NaN                      NaN   \n",
              "3             -0.037182                       NaN                      NaN   \n",
              "4              0.020397                       NaN                      NaN   \n",
              "\n",
              "   returnsOpenNextMktres10  universe  \n",
              "0                 0.034672       1.0  \n",
              "1                 0.027803       0.0  \n",
              "2                 0.024433       1.0  \n",
              "3                -0.007425       1.0  \n",
              "4                -0.017994       1.0  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "metadata": {
        "id": "e6mz7cWWrkW2",
        "colab_type": "code",
        "outputId": "1d145834-7d9f-4ab5-9c40-a581ce1e3ea1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "inputMktData = pd.DataFrame()\n",
        "inputMktData['date'] = marketDataframe['time']\n",
        "inputMktData['date'] =  pd.to_datetime(inputMktData['date'])\n",
        "inputMktData['date']=inputMktData['date'].dt.strftime('%d-%m-%Y')\n",
        "inputMktData['date'] =  pd.to_datetime(inputMktData['date'])\n",
        "inputMktData['assetName'] = marketDataframe['assetName']\n",
        "inputMktData['volume'] = marketDataframe['volume']\n",
        "inputMktData['close'] = marketDataframe['close']\n",
        "inputMktData['open'] = marketDataframe['open']\n",
        "inputNewsData = pd.DataFrame()\n",
        "inputNewsData['date'] = newsDataframe['time']\n",
        "inputNewsData['date'] =  pd.to_datetime(inputNewsData['date'])\n",
        "inputNewsData['date']=inputNewsData['date'].dt.strftime('%d-%m-%Y')\n",
        "inputNewsData['date'] =  pd.to_datetime(inputNewsData['date'])\n",
        "inputNewsData['headline'] = newsDataframe['headline']\n",
        "inputNewsData['urgency'] = newsDataframe['urgency']\n",
        "inputNewsData['subjects'] = newsDataframe['subjects']\n",
        "inputNewsData['audiences'] = newsDataframe['audiences']\n",
        "inputNewsData['audiences'] = newsDataframe['audiences']\n",
        "inputNewsData['assetName'] = newsDataframe['assetName']\n",
        "inputNewsData['relevance'] = newsDataframe['relevance']\n",
        "inputNewsData['sentimentNegative'] = newsDataframe['sentimentNegative']\n",
        "inputNewsData['sentimentNeutral'] = newsDataframe['sentimentNeutral']\n",
        "inputNewsData['sentimentPositive'] = newsDataframe['sentimentPositive']\n",
        "del newsDataframe,marketDataframe\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1min 39s, sys: 145 ms, total: 1min 40s\n",
            "Wall time: 1min 40s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "uoepJpTyVrQj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "inputNewsData = inputNewsData.dropna()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-rCVPDKgJKBV",
        "colab_type": "code",
        "outputId": "9b991b4d-d1bc-4ca7-ec0e-9abab53f0390",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "cell_type": "code",
      "source": [
        "inputNewsData.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>headline</th>\n",
              "      <th>urgency</th>\n",
              "      <th>subjects</th>\n",
              "      <th>audiences</th>\n",
              "      <th>assetName</th>\n",
              "      <th>relevance</th>\n",
              "      <th>sentimentNegative</th>\n",
              "      <th>sentimentNeutral</th>\n",
              "      <th>sentimentPositive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>China's Daqing pumps 43.41 mln tonnes of oil i...</td>\n",
              "      <td>3</td>\n",
              "      <td>{'ENR', 'ASIA', 'CN', 'NGS', 'EMRG', 'RTRS', '...</td>\n",
              "      <td>{'Z', 'O', 'OIL'}</td>\n",
              "      <td>PetroChina Co Ltd</td>\n",
              "      <td>0.235702</td>\n",
              "      <td>0.500739</td>\n",
              "      <td>0.419327</td>\n",
              "      <td>0.079934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>FEATURE-In kidnapping, finesse works best</td>\n",
              "      <td>3</td>\n",
              "      <td>{'FEA', 'CA', 'LATAM', 'MX', 'INS', 'ASIA', 'I...</td>\n",
              "      <td>{'PGE', 'PCO', 'G', 'ESN', 'MD', 'PCU', 'DNP',...</td>\n",
              "      <td>Travelers Companies Inc</td>\n",
              "      <td>0.447214</td>\n",
              "      <td>0.600082</td>\n",
              "      <td>0.345853</td>\n",
              "      <td>0.054064</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>PRESS DIGEST - Wall Street Journal - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>{'RET', 'ENR', 'ID', 'BG', 'US', 'PRESS', 'IQ'...</td>\n",
              "      <td>{'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...</td>\n",
              "      <td>Wal-Mart Stores Inc</td>\n",
              "      <td>0.377964</td>\n",
              "      <td>0.450049</td>\n",
              "      <td>0.295671</td>\n",
              "      <td>0.254280</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>PRESS DIGEST - New York Times - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>{'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...</td>\n",
              "      <td>{'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...</td>\n",
              "      <td>Google Inc</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>0.752917</td>\n",
              "      <td>0.162715</td>\n",
              "      <td>0.084368</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>PRESS DIGEST - New York Times - Jan 1</td>\n",
              "      <td>3</td>\n",
              "      <td>{'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...</td>\n",
              "      <td>{'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...</td>\n",
              "      <td>XM Satellite Radio Holdings Inc</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>0.699274</td>\n",
              "      <td>0.209360</td>\n",
              "      <td>0.091367</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        date                                           headline  urgency  \\\n",
              "0 2007-01-01  China's Daqing pumps 43.41 mln tonnes of oil i...        3   \n",
              "1 2007-01-01          FEATURE-In kidnapping, finesse works best        3   \n",
              "2 2007-01-01         PRESS DIGEST - Wall Street Journal - Jan 1        3   \n",
              "3 2007-01-01              PRESS DIGEST - New York Times - Jan 1        3   \n",
              "4 2007-01-01              PRESS DIGEST - New York Times - Jan 1        3   \n",
              "\n",
              "                                            subjects  \\\n",
              "0  {'ENR', 'ASIA', 'CN', 'NGS', 'EMRG', 'RTRS', '...   \n",
              "1  {'FEA', 'CA', 'LATAM', 'MX', 'INS', 'ASIA', 'I...   \n",
              "2  {'RET', 'ENR', 'ID', 'BG', 'US', 'PRESS', 'IQ'...   \n",
              "3  {'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...   \n",
              "4  {'FUND', 'FIN', 'CA', 'SFWR', 'INS', 'PUB', 'B...   \n",
              "\n",
              "                                           audiences  \\\n",
              "0                                  {'Z', 'O', 'OIL'}   \n",
              "1  {'PGE', 'PCO', 'G', 'ESN', 'MD', 'PCU', 'DNP',...   \n",
              "2  {'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...   \n",
              "3  {'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...   \n",
              "4  {'T', 'DNP', 'PSC', 'U', 'D', 'M', 'RNP', 'PTD...   \n",
              "\n",
              "                         assetName  relevance  sentimentNegative  \\\n",
              "0                PetroChina Co Ltd   0.235702           0.500739   \n",
              "1          Travelers Companies Inc   0.447214           0.600082   \n",
              "2              Wal-Mart Stores Inc   0.377964           0.450049   \n",
              "3                       Google Inc   0.149071           0.752917   \n",
              "4  XM Satellite Radio Holdings Inc   0.149071           0.699274   \n",
              "\n",
              "   sentimentNeutral  sentimentPositive  \n",
              "0          0.419327           0.079934  \n",
              "1          0.345853           0.054064  \n",
              "2          0.295671           0.254280  \n",
              "3          0.162715           0.084368  \n",
              "4          0.209360           0.091367  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "6Gkp6RfTHZIa",
        "colab_type": "code",
        "outputId": "5c567552-c4ab-457c-ec6f-fb3ec7f87d82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "cell_type": "code",
      "source": [
        "inputMktData.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>assetName</th>\n",
              "      <th>volume</th>\n",
              "      <th>close</th>\n",
              "      <th>open</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2007-01-02</td>\n",
              "      <td>Agilent Technologies Inc</td>\n",
              "      <td>2606900.0</td>\n",
              "      <td>32.19</td>\n",
              "      <td>32.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2007-01-02</td>\n",
              "      <td>AirTran Holdings Inc</td>\n",
              "      <td>2051600.0</td>\n",
              "      <td>11.12</td>\n",
              "      <td>11.08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2007-01-02</td>\n",
              "      <td>Advance Auto Parts Inc</td>\n",
              "      <td>1164800.0</td>\n",
              "      <td>37.51</td>\n",
              "      <td>37.99</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2007-01-02</td>\n",
              "      <td>Apple Inc</td>\n",
              "      <td>23747329.0</td>\n",
              "      <td>84.74</td>\n",
              "      <td>86.23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2007-01-02</td>\n",
              "      <td>ABB Ltd</td>\n",
              "      <td>1208600.0</td>\n",
              "      <td>18.02</td>\n",
              "      <td>18.01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        date                 assetName      volume  close   open\n",
              "0 2007-01-02  Agilent Technologies Inc   2606900.0  32.19  32.17\n",
              "1 2007-01-02      AirTran Holdings Inc   2051600.0  11.12  11.08\n",
              "2 2007-01-02    Advance Auto Parts Inc   1164800.0  37.51  37.99\n",
              "3 2007-01-02                 Apple Inc  23747329.0  84.74  86.23\n",
              "4 2007-01-02                   ABB Ltd   1208600.0  18.02  18.01"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "metadata": {
        "id": "ipmS03X6iwzR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "newsHeadlineData = inputNewsData['headline'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eBEdG-rAz8u-",
        "colab_type": "code",
        "outputId": "084395f9-20ee-4928-999d-fd3b7fb6e720",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "tagged_data = [TaggedDocument(words=word_tokenize(str(_d).lower()), tags=[str(i)]) for i, _d in enumerate(newsHeadlineData)]\n",
        "max_epochs = 5\n",
        "vec_size = 25\n",
        "alpha = 0.025\n",
        "model = Doc2Vec(size=vec_size,\n",
        "                alpha=alpha, \n",
        "                min_alpha=0.00025,\n",
        "                min_count=1,\n",
        "                dm =1)\n",
        "model.build_vocab(tagged_data)\n",
        "for epoch in range(max_epochs):\n",
        "    print('iteration {0}'.format(epoch))\n",
        "    model.train(tagged_data,\n",
        "                total_examples=model.corpus_count,\n",
        "                epochs=model.iter)\n",
        "    model.alpha -= 0.0002\n",
        "    model.min_alpha = model.alpha\n",
        "\n",
        "model.save(\"d2v.model\")\n",
        "del tagged_data\n",
        "print(\"Model Saved\")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/models/doc2vec.py:570: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
            "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "iteration 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: DeprecationWarning: Call to deprecated `iter` (Attribute will be removed in 4.0.0, use self.epochs instead).\n",
            "  from ipykernel import kernelapp as app\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "iteration 1\n",
            "iteration 2\n",
            "iteration 3\n",
            "iteration 4\n",
            "Model Saved\n",
            "CPU times: user 22min 55s, sys: 6min 36s, total: 29min 32s\n",
            "Wall time: 19min 2s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "F10rgudXVDqp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def longestVector(arr):\n",
        "  k=-1\n",
        "  maximum = -2\n",
        "  for i in range(len(arr)):\n",
        "    k = len(arr.item(i))\n",
        "    if k>maximum:\n",
        "      maximum = k\n",
        "      index = i\n",
        "  regex = r\"[']([a-zA-z]*)[']\"\n",
        "  line = arr.item(index)\n",
        "  matchObj = re.findall(regex, line, re.M|re.I)\n",
        "  length = len(matchObj)\n",
        "  return length\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Cw9PDoHSVLVy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def setOfAllCategories(arr):\n",
        "  allCats = set()\n",
        "  regex = r\"[']([a-zA-z]*)[']\"\n",
        "  for i in range(len(arr)):\n",
        "    line = arr.item(i)\n",
        "    matchObj = re.findall(regex, line, re.M|re.I)\n",
        "    for j in matchObj:\n",
        "      allCats.add(j)\n",
        "  return allCats"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jo5B_rfUVPXh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def createDictVector(numCat,nors,vs,arr):\n",
        "  vectorFrame = pd.DataFrame(np.zeros((nors,vs)))\n",
        "  DictionaryOfCategory = dict(numCat)\n",
        "  DictCat = {v: k for k, v in DictionaryOfCategory.items()}\n",
        "  regex = r\"[']([a-zA-z]*)[']\"\n",
        "  for i in range(len(arr)):\n",
        "    line = arr.item(i)\n",
        "    matchObj = re.findall(regex, line, re.M|re.I)\n",
        "    k=0\n",
        "    for j in matchObj:\n",
        "      vectorFrame.at[i,k] = DictCat[j] \n",
        "      k = k+1\n",
        "  df = vectorFrame\n",
        "  df['features'] = df[df.columns.values].values.tolist()\n",
        "  vectorNpArray = df['features'].values\n",
        "  return vectorNpArray"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GYVSLX7FxiwG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def vectorization(obj):\n",
        "  print('Calculating the vector size')\n",
        "  vectorSize = longestVector(obj)\n",
        "  print('Creating a list of all unique category values')\n",
        "  categories = setOfAllCategories(obj)\n",
        "  categories = list(categories)\n",
        "  numberOfRows = len(obj)\n",
        "  print('Creating Numbered List of Categories')\n",
        "  numberedListOfCategories = enumerate(categories)\n",
        "  print('Making an array of feature')\n",
        "  featureVector = createDictVector(numberedListOfCategories,numberOfRows,vectorSize,obj)\n",
        "  print('Finished')\n",
        "  return featureVector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jtGdXmabnFLP",
        "colab_type": "code",
        "outputId": "f8722c26-00a7-44c6-fe2e-05d4b2e8d533",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "cell_type": "code",
      "source": [
        "subjectArray = inputNewsData['subjects'].values\n",
        "featureVectorSubject = vectorization(subjectArray)\n",
        "del subjectArray\n",
        "print(featureVectorSubject)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Calculating the vector size\n",
            "Creating a list of all unique category values\n",
            "Creating Numbered List of Categories\n",
            "Making an array of feature\n",
            "Finished\n",
            "[list([69.0, 2.0, 322.0, 568.0, 210.0, 477.0, 392.0, 13.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([458.0, 482.0, 136.0, 361.0, 376.0, 2.0, 277.0, 605.0, 469.0, 200.0, 164.0, 477.0, 552.0, 295.0, 392.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([389.0, 69.0, 117.0, 642.0, 200.0, 159.0, 491.0, 525.0, 568.0, 472.0, 296.0, 477.0, 392.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " ...\n",
            " list([85.0, 200.0, 274.0, 91.0, 375.0, 636.0, 477.0, 272.0, 392.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([368.0, 535.0, 628.0, 323.0, 119.0, 408.0, 124.0, 200.0, 477.0, 392.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([368.0, 535.0, 323.0, 119.0, 408.0, 124.0, 200.0, 477.0, 392.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Z9Rbg3Uv5gdV",
        "colab_type": "code",
        "outputId": "b74f7a1b-6af9-4558-b774-5b1bc2527c20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "cell_type": "code",
      "source": [
        "audiencesArray = inputNewsData['audiences'].values\n",
        "featureVectorAudience = vectorization(audiencesArray)\n",
        "del audiencesArray\n",
        "print(featureVectorAudience)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Calculating the vector size\n",
            "Creating a list of all unique category values\n",
            "Creating Numbered List of Categories\n",
            "Making an array of feature\n",
            "Finished\n",
            "[list([46.0, 16.0, 105.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([57.0, 79.0, 34.0, 63.0, 81.0, 87.0, 68.0, 7.0, 75.0, 26.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0, 107.0, 3.0, 93.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " ...\n",
            " list([3.0, 75.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([3.0, 75.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])\n",
            " list([3.0, 7.0, 75.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "b1hdGovTW7ry",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def string2VecParser(obj):\n",
        "  tempVector2 = []\n",
        "  for i in range(len(obj)):\n",
        "      stringConv = re.findall( r'([\\d]*[.][\\d]*)[,]', str(obj[i]))\n",
        "      temp = []\n",
        "      for j in stringConv:\n",
        "        k = float(j)\n",
        "        temp.append(k)\n",
        "      tempVector = np.array(temp)\n",
        "      tempVector2.append(tempVector)\n",
        "  finalVector = np.array(tempVector2)\n",
        "  return finalVector\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mP0FlGsLrW0p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def queryEngine(marketIndex, newsDate):\n",
        "  td = timedelta(days=1)\n",
        "  upper = -45\n",
        "  lower = 20\n",
        "  p = inputMktData.loc[(inputMktData['date'] <= newsDate - upper*td) & (inputMktData['assetName'] == marketIndex) & (inputMktData['date'] >= newsDate - lower*td)]\n",
        "  if p.empty:\n",
        "    return pd.Series((None,None,None,None,None,None))\n",
        "  labelFrameTemp = p[p['date'] >= newsDate]\n",
        "  labelFrameTemp = labelFrameTemp.reset_index()\n",
        "  if len(labelFrameTemp.index) < 5:\n",
        "    return pd.Series((None,None,None,None,None,None))\n",
        "  labelFrameTemp2 = pd.DataFrame()\n",
        "  labelFrameTemp2['volume']=labelFrameTemp['volume'] \n",
        "  labelFrameTemp2['open']=labelFrameTemp['open']\n",
        "  labelFrameTemp2['close']=labelFrameTemp['close']\n",
        "  labelFrame = labelFrameTemp2.T\n",
        "  labelFrame['labelTemp'] = labelFrame[labelFrame.columns.values].values.tolist()\n",
        "  featureFrameTemp = p[p['date'] < newsDate]\n",
        "  featureFrameTemp = featureFrameTemp.reset_index()\n",
        "  featureFrameTemp2 = pd.DataFrame()\n",
        "  featureFrameTemp2['volume']=featureFrameTemp['volume']\n",
        "  featureFrameTemp2['open']=featureFrameTemp['open']\n",
        "  featureFrameTemp2['close']=featureFrameTemp['close']\n",
        "  featureFrame = featureFrameTemp2.T\n",
        "  featureFrame['featureTemp'] = featureFrame[featureFrame.columns.values].values.tolist()\n",
        "  label = labelFrame['labelTemp'].values\n",
        "  feature = featureFrame['featureTemp'].values\n",
        "  return pd.Series((label[0][:5],label[1][:5],label[2][:5],feature[0],feature[1],feature[2])) #label[volume open close] and feature[volume open close]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ov8wHHu1lvqT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def assembler(obj):\n",
        "  assembledVector = pd.DataFrame()\n",
        "  assembledVector['Vector'] = obj[obj.columns.values].values.tolist()\n",
        "  return assembledVector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TukIMUhcNDsw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def docvec(obj):\n",
        "  obj = str(obj).lower()\n",
        "  data = word_tokenize(obj)\n",
        "  return model.infer_vector(data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kBSX75ywoKc-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def queryForPandas(row):\n",
        "  return queryEngine(str(row['assetName']), row['date'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gAigryy--O9i",
        "colab_type": "code",
        "outputId": "bade8d9f-a0be-4f43-8035-17e1c03d627a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 147
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "#news parsing\n",
        "inputNewsData['subjectFeatures'] = featureVectorSubject.tolist() #subject to vector\n",
        "print('Subject Vectorized...')\n",
        "del featureVectorSubject\n",
        "inputNewsData['audienceFeatures'] = featureVectorAudience.tolist() #audience to Vector\n",
        "del featureVectorAudience\n",
        "print('Audiences Vectorized...')\n",
        "print('Headlines Vectorization Started')\n",
        "inputNewsData['headlineVector'] = inputNewsData['headline'].apply(docvec) #headline to Vector\n",
        "del model\n",
        "print('Headlines Vectorized...')\n",
        "print('Creating Cleaner DataFrame')\n",
        "NewsDataframeFinal = pd.DataFrame()\n",
        "NewsDataframeFinal['date'] = inputNewsData['date'] \n",
        "NewsDataframeFinal['urgency'] = inputNewsData['urgency']\n",
        "NewsDataframeFinal['relevance'] = inputNewsData['relevance']\n",
        "NewsDataframeFinal['assetName'] = inputNewsData['assetName']\n",
        "NewsDataframeFinal['sentimentNegative'] = inputNewsData['sentimentNegative']\n",
        "NewsDataframeFinal['sentimentNeutral'] = inputNewsData['sentimentNeutral']\n",
        "NewsDataframeFinal['sentimentPositive'] = inputNewsData['sentimentPositive']\n",
        "NewsDataframeFinal['subjectFeatures'] = inputNewsData['subjectFeatures']\n",
        "NewsDataframeFinal['audienceFeatures'] = inputNewsData['audienceFeatures']\n",
        "NewsDataframeFinal['headlineVector'] = inputNewsData['headlineVector']\n",
        "del inputNewsData\n",
        "print('DataFrame Creation Complete...')\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Subject Vectorized...\n",
            "Audiences Vectorized...\n",
            "Headlines Vectorization Started\n",
            "Headlines Vectorized...\n",
            "Creating Cleaner DataFrame\n",
            "DataFrame Creation Complete...\n",
            "CPU times: user 3min 51s, sys: 0 ns, total: 3min 51s\n",
            "Wall time: 3min 51s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "I5BfINBJEM9Z",
        "colab_type": "code",
        "outputId": "1249e28e-0be7-4831-988f-487f36c2dbbc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "print('Query Engine Started...')\n",
        "NewsDataframeFinal['labelVolume'] = np.nan\n",
        "NewsDataframeFinal['labelOpen'] = np.nan\n",
        "NewsDataframeFinal['labelClose'] = np.nan\n",
        "NewsDataframeFinal['featureVolume'] = np.nan\n",
        "NewsDataframeFinal['featureOpen'] = np.nan\n",
        "NewsDataframeFinal['featureClose'] = np.nan\n",
        "NewsDataframeFinal[['labelVolume','labelOpen','labelClose','featureVolume','featureOpen','featureClose']] = NewsDataframeFinal[['labelVolume','labelOpen','labelClose','featureVolume','featureOpen','featureClose']].astype(object)\n",
        "NewsDataframeFinal[['labelVolume','labelOpen','labelClose','featureVolume','featureOpen','featureClose']] = NewsDataframeFinal.apply(queryForPandas, axis = 1)\n",
        "NewsDataframeFinal = NewsDataframeFinal[NewsDataframeFinal.notnull()]\n",
        "print('Query Engine Task Finished...')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Query Engine Started...\n",
            "Query Engine Task Finished...\n",
            "CPU times: user 2h 45min 8s, sys: 0 ns, total: 2h 45min 8s\n",
            "Wall time: 2h 45min 13s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wdGYRpEngQ7A",
        "colab_type": "code",
        "outputId": "b9132a3f-84fc-42ff-d526-10545eb645a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 471
        }
      },
      "cell_type": "code",
      "source": [
        "NewsDataframeFinal.head()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>urgency</th>\n",
              "      <th>relevance</th>\n",
              "      <th>assetName</th>\n",
              "      <th>sentimentNegative</th>\n",
              "      <th>sentimentNeutral</th>\n",
              "      <th>sentimentPositive</th>\n",
              "      <th>subjectFeatures</th>\n",
              "      <th>audienceFeatures</th>\n",
              "      <th>headlineVector</th>\n",
              "      <th>labelVolume</th>\n",
              "      <th>labelOpen</th>\n",
              "      <th>labelClose</th>\n",
              "      <th>featureVolume</th>\n",
              "      <th>featureOpen</th>\n",
              "      <th>featureClose</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.235702</td>\n",
              "      <td>PetroChina Co Ltd</td>\n",
              "      <td>0.500739</td>\n",
              "      <td>0.419327</td>\n",
              "      <td>0.079934</td>\n",
              "      <td>[69.0, 2.0, 322.0, 568.0, 210.0, 477.0, 392.0,...</td>\n",
              "      <td>[46.0, 16.0, 105.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0...</td>\n",
              "      <td>[-0.06354199, -0.03799604, -0.12417433, -0.012...</td>\n",
              "      <td>[838700.0, 687700.0, 628836.0, 665539.0, 40430...</td>\n",
              "      <td>[122.7, 124.44, 121.95, 121.95, 122.25]</td>\n",
              "      <td>[123.4, 124.15, 122.8, 122.88, 121.8]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.447214</td>\n",
              "      <td>Travelers Companies Inc</td>\n",
              "      <td>0.600082</td>\n",
              "      <td>0.345853</td>\n",
              "      <td>0.054064</td>\n",
              "      <td>[458.0, 482.0, 136.0, 361.0, 376.0, 2.0, 277.0...</td>\n",
              "      <td>[57.0, 79.0, 34.0, 63.0, 81.0, 87.0, 68.0, 7.0...</td>\n",
              "      <td>[0.09221249, -0.041277327, -0.033864953, -0.23...</td>\n",
              "      <td>[1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...</td>\n",
              "      <td>[51.27, 52.97, 54.8, 54.17, 53.73]</td>\n",
              "      <td>[51.85, 54.44, 54.6, 54.58, 54.48]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.377964</td>\n",
              "      <td>Wal-Mart Stores Inc</td>\n",
              "      <td>0.450049</td>\n",
              "      <td>0.295671</td>\n",
              "      <td>0.254280</td>\n",
              "      <td>[389.0, 69.0, 117.0, 642.0, 200.0, 159.0, 491....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[0.08623818, 0.060358655, 0.011216616, -0.0078...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>Google Inc</td>\n",
              "      <td>0.752917</td>\n",
              "      <td>0.162715</td>\n",
              "      <td>0.084368</td>\n",
              "      <td>[22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[0.0030525122, 0.0105378665, -0.0068828673, 0....</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>XM Satellite Radio Holdings Inc</td>\n",
              "      <td>0.699274</td>\n",
              "      <td>0.209360</td>\n",
              "      <td>0.091367</td>\n",
              "      <td>[22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[-0.035201706, 0.051901802, -0.0036501635, -0....</td>\n",
              "      <td>[6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...</td>\n",
              "      <td>[14.14, 14.15, 13.01, 13.18, 13.0]</td>\n",
              "      <td>[14.15, 14.21, 13.06, 13.22, 12.98]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        date  urgency  relevance                        assetName  \\\n",
              "0 2007-01-01        3   0.235702                PetroChina Co Ltd   \n",
              "1 2007-01-01        3   0.447214          Travelers Companies Inc   \n",
              "2 2007-01-01        3   0.377964              Wal-Mart Stores Inc   \n",
              "3 2007-01-01        3   0.149071                       Google Inc   \n",
              "4 2007-01-01        3   0.149071  XM Satellite Radio Holdings Inc   \n",
              "\n",
              "   sentimentNegative  sentimentNeutral  sentimentPositive  \\\n",
              "0           0.500739          0.419327           0.079934   \n",
              "1           0.600082          0.345853           0.054064   \n",
              "2           0.450049          0.295671           0.254280   \n",
              "3           0.752917          0.162715           0.084368   \n",
              "4           0.699274          0.209360           0.091367   \n",
              "\n",
              "                                     subjectFeatures  \\\n",
              "0  [69.0, 2.0, 322.0, 568.0, 210.0, 477.0, 392.0,...   \n",
              "1  [458.0, 482.0, 136.0, 361.0, 376.0, 2.0, 277.0...   \n",
              "2  [389.0, 69.0, 117.0, 642.0, 200.0, 159.0, 491....   \n",
              "3  [22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....   \n",
              "4  [22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....   \n",
              "\n",
              "                                    audienceFeatures  \\\n",
              "0  [46.0, 16.0, 105.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0...   \n",
              "1  [57.0, 79.0, 34.0, 63.0, 81.0, 87.0, 68.0, 7.0...   \n",
              "2  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "3  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "4  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "\n",
              "                                      headlineVector  \\\n",
              "0  [-0.06354199, -0.03799604, -0.12417433, -0.012...   \n",
              "1  [0.09221249, -0.041277327, -0.033864953, -0.23...   \n",
              "2  [0.08623818, 0.060358655, 0.011216616, -0.0078...   \n",
              "3  [0.0030525122, 0.0105378665, -0.0068828673, 0....   \n",
              "4  [-0.035201706, 0.051901802, -0.0036501635, -0....   \n",
              "\n",
              "                                         labelVolume  \\\n",
              "0  [838700.0, 687700.0, 628836.0, 665539.0, 40430...   \n",
              "1  [1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...   \n",
              "2                                                NaN   \n",
              "3                                                NaN   \n",
              "4  [6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...   \n",
              "\n",
              "                                 labelOpen  \\\n",
              "0  [122.7, 124.44, 121.95, 121.95, 122.25]   \n",
              "1       [51.27, 52.97, 54.8, 54.17, 53.73]   \n",
              "2                                      NaN   \n",
              "3                                      NaN   \n",
              "4       [14.14, 14.15, 13.01, 13.18, 13.0]   \n",
              "\n",
              "                              labelClose featureVolume featureOpen  \\\n",
              "0  [123.4, 124.15, 122.8, 122.88, 121.8]            []          []   \n",
              "1     [51.85, 54.44, 54.6, 54.58, 54.48]            []          []   \n",
              "2                                    NaN           NaN         NaN   \n",
              "3                                    NaN           NaN         NaN   \n",
              "4    [14.15, 14.21, 13.06, 13.22, 12.98]            []          []   \n",
              "\n",
              "  featureClose  \n",
              "0           []  \n",
              "1           []  \n",
              "2          NaN  \n",
              "3          NaN  \n",
              "4           []  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "SqM-NOhySx3J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "outputId": "b1d41745-a1e7-4c37-8119-eec7a027aeb9"
      },
      "cell_type": "code",
      "source": [
        "NewsDataframeFinal = NewsDataframeFinal[NewsDataframeFinal.notnull()]\n",
        "NewsDataframeFinal = NewsDataframeFinal.dropna()\n",
        "NewsDataframeFinal.head()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>urgency</th>\n",
              "      <th>relevance</th>\n",
              "      <th>assetName</th>\n",
              "      <th>sentimentNegative</th>\n",
              "      <th>sentimentNeutral</th>\n",
              "      <th>sentimentPositive</th>\n",
              "      <th>subjectFeatures</th>\n",
              "      <th>audienceFeatures</th>\n",
              "      <th>headlineVector</th>\n",
              "      <th>labelVolume</th>\n",
              "      <th>labelOpen</th>\n",
              "      <th>labelClose</th>\n",
              "      <th>featureVolume</th>\n",
              "      <th>featureOpen</th>\n",
              "      <th>featureClose</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.235702</td>\n",
              "      <td>PetroChina Co Ltd</td>\n",
              "      <td>0.500739</td>\n",
              "      <td>0.419327</td>\n",
              "      <td>0.079934</td>\n",
              "      <td>[69.0, 2.0, 322.0, 568.0, 210.0, 477.0, 392.0,...</td>\n",
              "      <td>[46.0, 16.0, 105.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0...</td>\n",
              "      <td>[-0.06354199, -0.03799604, -0.12417433, -0.012...</td>\n",
              "      <td>[838700.0, 687700.0, 628836.0, 665539.0, 40430...</td>\n",
              "      <td>[122.7, 124.44, 121.95, 121.95, 122.25]</td>\n",
              "      <td>[123.4, 124.15, 122.8, 122.88, 121.8]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.447214</td>\n",
              "      <td>Travelers Companies Inc</td>\n",
              "      <td>0.600082</td>\n",
              "      <td>0.345853</td>\n",
              "      <td>0.054064</td>\n",
              "      <td>[458.0, 482.0, 136.0, 361.0, 376.0, 2.0, 277.0...</td>\n",
              "      <td>[57.0, 79.0, 34.0, 63.0, 81.0, 87.0, 68.0, 7.0...</td>\n",
              "      <td>[0.09221249, -0.041277327, -0.033864953, -0.23...</td>\n",
              "      <td>[1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...</td>\n",
              "      <td>[51.27, 52.97, 54.8, 54.17, 53.73]</td>\n",
              "      <td>[51.85, 54.44, 54.6, 54.58, 54.48]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>XM Satellite Radio Holdings Inc</td>\n",
              "      <td>0.699274</td>\n",
              "      <td>0.209360</td>\n",
              "      <td>0.091367</td>\n",
              "      <td>[22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[-0.035201706, 0.051901802, -0.0036501635, -0....</td>\n",
              "      <td>[6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...</td>\n",
              "      <td>[14.14, 14.15, 13.01, 13.18, 13.0]</td>\n",
              "      <td>[14.15, 14.21, 13.06, 13.22, 12.98]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.149071</td>\n",
              "      <td>Walt Disney Co</td>\n",
              "      <td>0.152658</td>\n",
              "      <td>0.615878</td>\n",
              "      <td>0.231464</td>\n",
              "      <td>[22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[0.010929393, 0.01853853, 0.015210629, -0.0617...</td>\n",
              "      <td>[8048100.0, 7670300.0, 12848592.0, 11835125.0,...</td>\n",
              "      <td>[35.09, 35.02, 34.07, 34.58, 34.7]</td>\n",
              "      <td>[35.01, 35.18, 34.59, 34.75, 34.67]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2007-01-01</td>\n",
              "      <td>3</td>\n",
              "      <td>0.447214</td>\n",
              "      <td>Microsoft Corp</td>\n",
              "      <td>0.325929</td>\n",
              "      <td>0.374173</td>\n",
              "      <td>0.299898</td>\n",
              "      <td>[22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....</td>\n",
              "      <td>[89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...</td>\n",
              "      <td>[4.1332096e-06, 0.017256232, 0.024086958, -0.0...</td>\n",
              "      <td>[55367062.0, 60444038.0, 51763813.0, 55591266....</td>\n",
              "      <td>[30.84, 30.825, 29.045, 29.165, 29.57]</td>\n",
              "      <td>[30.56, 30.19, 29.03, 29.4, 29.46]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        date  urgency  relevance                        assetName  \\\n",
              "0 2007-01-01        3   0.235702                PetroChina Co Ltd   \n",
              "1 2007-01-01        3   0.447214          Travelers Companies Inc   \n",
              "4 2007-01-01        3   0.149071  XM Satellite Radio Holdings Inc   \n",
              "6 2007-01-01        3   0.149071                   Walt Disney Co   \n",
              "7 2007-01-01        3   0.447214                   Microsoft Corp   \n",
              "\n",
              "   sentimentNegative  sentimentNeutral  sentimentPositive  \\\n",
              "0           0.500739          0.419327           0.079934   \n",
              "1           0.600082          0.345853           0.054064   \n",
              "4           0.699274          0.209360           0.091367   \n",
              "6           0.152658          0.615878           0.231464   \n",
              "7           0.325929          0.374173           0.299898   \n",
              "\n",
              "                                     subjectFeatures  \\\n",
              "0  [69.0, 2.0, 322.0, 568.0, 210.0, 477.0, 392.0,...   \n",
              "1  [458.0, 482.0, 136.0, 361.0, 376.0, 2.0, 277.0...   \n",
              "4  [22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....   \n",
              "6  [22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....   \n",
              "7  [22.0, 486.0, 482.0, 538.0, 376.0, 369.0, 119....   \n",
              "\n",
              "                                    audienceFeatures  \\\n",
              "0  [46.0, 16.0, 105.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0...   \n",
              "1  [57.0, 79.0, 34.0, 63.0, 81.0, 87.0, 68.0, 7.0...   \n",
              "4  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "6  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "7  [89.0, 68.0, 7.0, 75.0, 44.0, 20.0, 26.0, 18.0...   \n",
              "\n",
              "                                      headlineVector  \\\n",
              "0  [-0.06354199, -0.03799604, -0.12417433, -0.012...   \n",
              "1  [0.09221249, -0.041277327, -0.033864953, -0.23...   \n",
              "4  [-0.035201706, 0.051901802, -0.0036501635, -0....   \n",
              "6  [0.010929393, 0.01853853, 0.015210629, -0.0617...   \n",
              "7  [4.1332096e-06, 0.017256232, 0.024086958, -0.0...   \n",
              "\n",
              "                                         labelVolume  \\\n",
              "0  [838700.0, 687700.0, 628836.0, 665539.0, 40430...   \n",
              "1  [1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...   \n",
              "4  [6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...   \n",
              "6  [8048100.0, 7670300.0, 12848592.0, 11835125.0,...   \n",
              "7  [55367062.0, 60444038.0, 51763813.0, 55591266....   \n",
              "\n",
              "                                 labelOpen  \\\n",
              "0  [122.7, 124.44, 121.95, 121.95, 122.25]   \n",
              "1       [51.27, 52.97, 54.8, 54.17, 53.73]   \n",
              "4       [14.14, 14.15, 13.01, 13.18, 13.0]   \n",
              "6       [35.09, 35.02, 34.07, 34.58, 34.7]   \n",
              "7   [30.84, 30.825, 29.045, 29.165, 29.57]   \n",
              "\n",
              "                              labelClose featureVolume featureOpen  \\\n",
              "0  [123.4, 124.15, 122.8, 122.88, 121.8]            []          []   \n",
              "1     [51.85, 54.44, 54.6, 54.58, 54.48]            []          []   \n",
              "4    [14.15, 14.21, 13.06, 13.22, 12.98]            []          []   \n",
              "6    [35.01, 35.18, 34.59, 34.75, 34.67]            []          []   \n",
              "7     [30.56, 30.19, 29.03, 29.4, 29.46]            []          []   \n",
              "\n",
              "  featureClose  \n",
              "0           []  \n",
              "1           []  \n",
              "4           []  \n",
              "6           []  \n",
              "7           []  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "leMsCTLQtAOf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def flat_list(obj):  \n",
        "  listVal = [y for x in obj for y in x]\n",
        "  return listVal"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-t0RwVDvqTF-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Data = pd.DataFrame()\n",
        "Data['label'] = NewsDataframeFinal[['labelVolume','labelOpen','labelClose']].values.tolist()\n",
        "Data['sentimentFeature'] = NewsDataframeFinal[['urgency','relevance','sentimentPositive','sentimentNeutral','sentimentNegative']].values.tolist()\n",
        "Data['newsFeature'] = NewsDataframeFinal[['subjectFeatures','audienceFeatures','headlineVector','featureVolume','featureOpen','featureClose']].values.tolist()\n",
        "Data['newsFeature'] = Data['newsFeature'].apply(flat_list)\n",
        "del NewsDataframeFinal\n",
        "Data['features'] = Data[['sentimentFeature','newsFeature']].values.tolist()\n",
        "Data = Data.drop(columns=['sentimentFeature','newsFeature'])\n",
        "Data['label'] = Data['label'].apply(flat_list)\n",
        "Data['features'] = Data['features'].apply(flat_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WYpSD-VpZvb0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "outputId": "f9cbeeef-e9e4-4770-9839-9c846df907e6"
      },
      "cell_type": "code",
      "source": [
        "Data.head()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>features</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[838700.0, 687700.0, 628836.0, 665539.0, 40430...</td>\n",
              "      <td>[3.0, 0.235702, 0.0799341, 0.41932700000000006...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...</td>\n",
              "      <td>[3.0, 0.44721400000000006, 0.0540643, 0.345853...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...</td>\n",
              "      <td>[3.0, 0.149071, 0.0913665, 0.20936, 0.699274, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[8048100.0, 7670300.0, 12848592.0, 11835125.0,...</td>\n",
              "      <td>[3.0, 0.149071, 0.23146399999999998, 0.6158779...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[55367062.0, 60444038.0, 51763813.0, 55591266....</td>\n",
              "      <td>[3.0, 0.44721400000000006, 0.299898, 0.374173,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               label  \\\n",
              "0  [838700.0, 687700.0, 628836.0, 665539.0, 40430...   \n",
              "1  [1513000.0, 2626171.0, 2128988.0, 2679145.0, 2...   \n",
              "2  [6452689.0, 5096016.0, 6724300.0, 7297105.0, 7...   \n",
              "3  [8048100.0, 7670300.0, 12848592.0, 11835125.0,...   \n",
              "4  [55367062.0, 60444038.0, 51763813.0, 55591266....   \n",
              "\n",
              "                                            features  \n",
              "0  [3.0, 0.235702, 0.0799341, 0.41932700000000006...  \n",
              "1  [3.0, 0.44721400000000006, 0.0540643, 0.345853...  \n",
              "2  [3.0, 0.149071, 0.0913665, 0.20936, 0.699274, ...  \n",
              "3  [3.0, 0.149071, 0.23146399999999998, 0.6158779...  \n",
              "4  [3.0, 0.44721400000000006, 0.299898, 0.374173,...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "metadata": {
        "id": "byqDhh4wEEHp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "99306aff-d750-448a-92bb-fca131126640"
      },
      "cell_type": "code",
      "source": [
        "import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Activation, LSTM, Conv1D, MaxPooling1D, Embedding\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing import sequence\n",
        "from sklearn.preprocessing import normalize\n",
        "\n",
        "features = Data['features'].values\n",
        "labels = Data['label'].values"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "JpNHaOEVXxxH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "cff41156-7482-4ca5-846a-d957cb405100"
      },
      "cell_type": "code",
      "source": [
        "#hyperparameters\n",
        "pool_size = 15\n",
        "lstm_output_size = 15\n",
        "batch_size = 10000\n",
        "epochs = 5\n",
        "max_features = 10000\n",
        "maxlen = 200\n",
        "embedding_size = 160\n",
        "\n",
        "print('Loading data...')\n",
        "x_train, x_test, y_train, y_test = train_test_split(features,labels, test_size= 0.2, random_state=21)\n",
        "x_train = sequence.pad_sequences(x_train, maxlen=maxlen, padding='post')\n",
        "x_test = sequence.pad_sequences(x_test, maxlen=maxlen, padding='post')\n",
        "y_train = sequence.pad_sequences(x_train, maxlen=15, padding='post')\n",
        "y_test = sequence.pad_sequences(x_test, maxlen=15, padding='post')\n",
        "x_train = normalize(x_train)\n",
        "y_train = normalize(y_train)\n",
        "x_test = normalize(x_test)\n",
        "y_test = normalize(y_test)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "3zqOm1J2n4H0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        },
        "outputId": "fe018b68-b251-4ea4-a7db-e1f9f7449ee2"
      },
      "cell_type": "code",
      "source": [
        "print('Building model...')\n",
        "model = Sequential()\n",
        "model.add(Embedding(max_features, embedding_size, input_length=maxlen))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Conv1D (kernel_size = (5), filters = 40, activation='relu'))\n",
        "model.add(MaxPooling1D(pool_size=pool_size))\n",
        "model.add(LSTM(lstm_output_size))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(15)) #prediction for next five days so the label should be made such that it has to be 15 dimensional(volume open close 5 days)\n",
        "model.add(Activation('sigmoid'))\n",
        "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
        "\n",
        "print('Train...')\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=batch_size,\n",
        "          epochs=epochs,\n",
        "          validation_data=(x_test, y_test))\n",
        "score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)\n",
        "model.save('drive/My Drive/stockNewsDataset/my_model.hdf5')\n",
        "print('Test score:', score)\n",
        "print('Test accuracy:', acc)\n"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building model...\n",
            "Train...\n",
            "Train on 181728 samples, validate on 45432 samples\n",
            "Epoch 1/5\n",
            "181728/181728 [==============================] - 22s 119us/step - loss: 0.6274 - acc: 0.7051 - val_loss: 0.5148 - val_acc: 0.9328\n",
            "Epoch 2/5\n",
            "181728/181728 [==============================] - 17s 92us/step - loss: 0.4526 - acc: 0.8271 - val_loss: 0.3162 - val_acc: 0.9328\n",
            "Epoch 3/5\n",
            "181728/181728 [==============================] - 17s 92us/step - loss: 0.3270 - acc: 0.8861 - val_loss: 0.2169 - val_acc: 0.9328\n",
            "Epoch 4/5\n",
            "181728/181728 [==============================] - 17s 92us/step - loss: 0.2555 - acc: 0.9224 - val_loss: 0.1632 - val_acc: 0.9995\n",
            "Epoch 5/5\n",
            "181728/181728 [==============================] - 17s 92us/step - loss: 0.2063 - acc: 0.9450 - val_loss: 0.1267 - val_acc: 0.9995\n",
            "45432/45432 [==============================] - 1s 21us/step\n",
            "Test score: 0.12665072754143536\n",
            "Test accuracy: 0.999474684943152\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Hfn2tAIscgj-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}