split_text_attempt.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNSP5MsjpgIL6V07c3QEUen",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/YanzhaoZ/PreBit/blob/main/split_text_attempt.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87zKeMTmYZVM",
        "outputId": "ae327c9d-fb9a-4148-efa1-04bad2ee08c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        }
      },
      "source": [
        "import pandas as pd\n",
        "import glob\n",
        "import re\n",
        "import nltk\n",
        "nltk.download('words')\n",
        "\n",
        "!pip install yfinance\n",
        "import yfinance as yf\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Package words is already up-to-date!\n",
            "Requirement already satisfied: yfinance in /usr/local/lib/python3.6/dist-packages (0.1.55)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from yfinance) (0.0.9)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from yfinance) (1.18.5)\n",
            "Requirement already satisfied: lxml>=4.5.1 in /usr/local/lib/python3.6/dist-packages (from yfinance) (4.6.1)\n",
            "Requirement already satisfied: pandas>=0.24 in /usr/local/lib/python3.6/dist-packages (from yfinance) (1.1.2)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.6/dist-packages (from yfinance) (2.23.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24->yfinance) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24->yfinance) (2018.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20->yfinance) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20->yfinance) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20->yfinance) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20->yfinance) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas>=0.24->yfinance) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "32tjIeFcYi8e"
      },
      "source": [
        "# Loading in data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_107ymtYqLF",
        "outputId": "a93fd46b-2240-4ec3-c34e-f43261df32c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aF9y_Fj5bCV7"
      },
      "source": [
        "#now that I have the whole string for the day, I need to do processing first, before spliting them into chunks. \n",
        "words = set(nltk.corpus.words.words())\n",
        "#I need to add in some crypto currency specific words. like Etherum.\n",
        "#bascailly all the symbols of crypto currencies, I should add a set here. \n",
        "cryptowords=(['crypto','cryptocurrency','bitcoin','btc','etherum','eos','eth','xrp','ltc'])\n",
        "words.update(cryptowords)\n",
        "\n",
        "exclusionli=['up','so','if','go'] #keep in view\n",
        "\n",
        "\n",
        "def processTweet(tweetFeed):\n",
        "    tweetFeed = tweetFeed.lower()\n",
        "    #Convert www.* or https?://* pic.twitter.com* to space\n",
        "    tweetFeed = re.sub('(https:\\/\\/www\\.[\\s][\\a-zA-Z]*)|(http:\\/\\/www\\.[\\s][\\a-zA-Z]*)|(https:\\/\\/[\\s][\\a-zA-Z]*)|(http:\\/\\/[\\s][\\a-zA-Z]*)|(www\\.[\\s][\\a-zA-Z]*)|(https:\\/\\/[\\a-zA-Z]*)|(http:\\/\\/[\\a-zA-Z]*)',' ',tweetFeed)\n",
        "    tweetFeed = re.sub('pic\\.[^\\s]*',' ',tweetFeed)\n",
        "    #removw @sth and #sth\n",
        "    tweetFeed = re.sub('(@\\ [^\\s]*)|(#\\ [^\\s]*)', ' ', tweetFeed)\n",
        "    #remove rt\n",
        "    tweetFeed = re.sub('rt[\\s]','',tweetFeed)\n",
        "\n",
        "    tweetFeed = \" \".join(w for w in nltk.wordpunct_tokenize(tweetFeed) if w.lower() in words)# or not w.isalpha())#this part takes in number as True.\n",
        "    #I'm not so sure about this language filter, but let's just roll with it first. \n",
        "    tweetFeed = re.sub(\"[^A-Za-z']+\", ' ', tweetFeed)\n",
        "\n",
        "    #filter out all the token left with len of 1. possibaly 2, defind a exclusion list: up, if, so, \n",
        "    tweetFeed = \" \".join(w for w in nltk.wordpunct_tokenize(tweetFeed) if len(w) >1 )\n",
        "    return tweetFeed\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fuaaoRnkIeKA"
      },
      "source": [
        "#now let's try to split the text into smaller chunks, which is just taking the tutorial's code\n",
        "def get_split(text1,length=200, overlap=50):\n",
        "  l_total = []\n",
        "  l_parcial = []\n",
        "  if len(text1.split())//(length-overlap) >0:\n",
        "    n = len(text1.split())//(length-overlap)\n",
        "  else: \n",
        "    n = 1\n",
        "  for w in range(n):\n",
        "    if w == 0:\n",
        "      l_parcial = text1.split()[:length]\n",
        "      l_total.append(\" \".join(l_parcial))\n",
        "    else:\n",
        "      l_parcial = text1.split()[w*(length-overlap):w*(length-overlap) + length]\n",
        "      l_total.append(\" \".join(l_parcial))\n",
        "  return l_total"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Hzv5dJJL73E"
      },
      "source": [
        "#next, tie them up, read in csv, combine into chunks, take the dates as the price? \n",
        "path = '/content/gdrive/My Drive/Crypto/tweets2019'\n",
        "all_files = glob.glob(path+'/*.csv')\n",
        "\n",
        "#let me try to use list1 to store string and list 2 to store dates\n",
        "li1=[]\n",
        "li2=[]\n",
        "for file in all_files:\n",
        "  df=pd.read_csv(file,sep=',',index_col=0)\n",
        "  df=df['text'].to_frame()\n",
        "  rawtxt=[]\n",
        "  for row in df.text:\n",
        "    rawtxt.append(row)\n",
        "  string = \" \".join(rawtxt)\n",
        "\n",
        "  date= file[-14:-4]\n",
        "  \n",
        "  li1.append(string)\n",
        "  li2.append(date)\n",
        "\n",
        "data=pd.DataFrame({'date':li2, 'text':li1})\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L2pWPWDYMs1R",
        "outputId": "f881a78a-04ce-4282-cdb3-7ab19a332906",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#now, let's process the text with the pre-defined functin. and perhaps add in the label by looking at the price. \n",
        "data['clean_txt']=data['text'].apply(processTweet)\n",
        "\n",
        "#A note for the price, file date is the day after all the tweets inside. So the label should be created, as such that it reflex:\n",
        "#file date over previous day change. \n",
        "#What index should I use/should I use my own file (maybe not, too big/detailed for daily ticker). Let's go with Yahoofinance.\n",
        "#now, let's create the price change label, return diff, log return diff, and binary label \n",
        "start_date ='2018-12-25'\n",
        "end_date = '2019-01-15'\n",
        "price = yf.download(\"BTC-USD\", start=start_date, end=end_date)\n",
        "price_2= price.Close.to_frame()\n",
        "price['change']=price_2.apply(lambda x: x/x.shift(1)-1)\n",
        "price['log_change']=price_2.apply(lambda x: np.log(x)-np.log(x.shift(1)))\n",
        "price['change_label']=price['change'].apply (lambda x: x>0)\n",
        "price=price.reset_index()\n",
        "price['date']=price.Date.apply (lambda x : str(x)[:10])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKo8j3GZMt49"
      },
      "source": [
        "#price.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BL6PdrjcY7Nt"
      },
      "source": [
        "#now that I have everything, I need to join the price set with my text set, by the date column. \n",
        "#I have a problem with datetime type on price frame vs string in text date, which converstion should I use, convert to string?  Let\n",
        "# us convert datetime to string first. if need to, we can do the otherway in the future. \n",
        "# import datetime\n",
        "# t = datetime.datetime(2012, 2, 23, 0, 0)\n",
        "# t.strftime('%Y-%m-%d')\n",
        "\n",
        "frame_x = data.merge(price, how='outer', on=['date'])\n",
        "#and drop any non-overlapping cells by dropna.\n",
        "frame_x = frame_x.dropna()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSLwAXzK4cCl"
      },
      "source": [
        "#frame_x.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2LmoPhFLzngF"
      },
      "source": [
        "#now that I have the frame prepared, I think I should do the text splitting part? \n",
        "#I need to decide on the length to use, and embedding to use. \n",
        "#let's roll with the 200 length for now. \n",
        "\n",
        "frame_x['text_split']=frame_x.clean_txt.apply(get_split)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vN4t5acu1kvi"
      },
      "source": [
        "#frame_x.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yECEmzvEn6zw",
        "outputId": "cc4c753f-28bf-4aec-d1ee-63642f1b2d5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(frame_x['text_split'][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "280"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JDaHaOt-7GVE"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EldlobMM7ZjH"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JinMz2dU5rpS"
      },
      "source": [
        "#try to save the processed frame to load directly next time\n",
        "df_save=frame_x[['date','text_split','change_label']]\n",
        "df_save.to_csv('/content/gdrive/My Drive/Crypto/df_save.csv',index=False)\n",
        "df_save.to_csv('/content/gdrive/My Drive/Crypto/df2_save.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQPbslBP6jiS",
        "outputId": "e169c7f0-4b22-45db-b7ac-336e9f628684",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df=pd.read_csv('/content/gdrive/My Drive/Crypto/df_save.csv',sep=',')\n",
        "df.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>text_split</th>\n",
              "      <th>change_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2019-01-11</td>\n",
              "      <td>['btc bitcoin current price more on crypto vis...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2019-01-10</td>\n",
              "      <td>['top crypto news for today from bitcoin and k...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2019-01-04</td>\n",
              "      <td>['na data de era do da btc san bitcoin how bit...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2019-01-06</td>\n",
              "      <td>['btc in ripple and xrp to become payment proc...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>2019-01-07</td>\n",
              "      <td>['watch how to buy bitcoin the safe and easy w...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         date                                         text_split  change_label\n",
              "5  2019-01-11  ['btc bitcoin current price more on crypto vis...          True\n",
              "6  2019-01-10  ['top crypto news for today from bitcoin and k...         False\n",
              "7  2019-01-04  ['na data de era do da btc san bitcoin how bit...          True\n",
              "8  2019-01-06  ['btc in ripple and xrp to become payment proc...          True\n",
              "9  2019-01-07  ['watch how to buy bitcoin the safe and easy w...         False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ME1imadJjAd",
        "outputId": "5abb062a-dc58-48f0-b9ca-6d1dc62427ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#ok, now, let's save another version, with every chunk as an entry. \n",
        "#this part, omits the dates signal for now. btw \n",
        "train_l = []\n",
        "label_l = []\n",
        "index_l =[]\n",
        "for idx,row in df_save.iterrows():\n",
        "  for l in row['text_split']:\n",
        "    train_l.append(l)\n",
        "    label_l.append(row['change_label'])\n",
        "    index_l.append(idx)\n",
        "len(train_l), len(label_l), len(index_l)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2718, 2718, 2718)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77tWanhgKfjX"
      },
      "source": [
        "#converting the label from True/False to 1/0. might be redundant, see how. \n",
        "label_l_num= [1 if row == True else 0 for row in label_l]\n",
        "\n",
        "train_df = pd.DataFrame({'text_split':train_l, 'change_label':label_l_num})\n",
        "train_df.to_csv('/content/gdrive/My Drive/Crypto/df3_save.csv',index=False)\n",
        "train_df.to_csv('/content/gdrive/My Drive/Crypto/df4_save.csv',index=False)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hbgB9AxTUBmv"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KWNugF4UBdX",
        "outputId": "2411e3b6-641f-4332-d341-eddcd3c92054",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "train_df.isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "text_split      0\n",
              "change_label    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "co7CLtSi779J",
        "outputId": "a2437075-05ca-4061-96bc-dee54e578543",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!ls '/content/gdrive/My Drive/Crypto'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "df_save.csv\t      GoogleNews-vectors-negative300.bin.gz\n",
            "final_price_2019.csv  tweets2019\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G-bNtwe-vPxt"
      },
      "source": [
        "# Get word embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NE_uDu-avV5N"
      },
      "source": [
        "#ok text and labels are somewhat prepared, time to bring out the big guns,get the embedding. \n",
        "#USE PRETRAINED ONES, what embeddings to use. doesn't have to be BERT. \n",
        "\n",
        "#in the BERT tutorial mode, there are a few steps:\n",
        "#1. prepare data for Bert\n",
        "#2. finetune Bert\n",
        "#3. extract Bert embedding\n",
        "#4. prepare the emedding to LSTM, which takes in variable input length, gotta do padding and masking and stuff. \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fb7XFWDhJUAJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFBUK65SvZvY"
      },
      "source": [
        "import torch\n",
        "from torchtext import data\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SnOknTdzWsCj"
      },
      "source": [
        "TEXT = data.Field(tokenize = 'spacy', include_lengths = True)\n",
        "LABEL = data.LabelField(dtype = torch.float)\n",
        "\n",
        "from torchtext import datasets\n",
        "\n",
        "#train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4mQM5TIWr-c",
        "outputId": "3b90d088-efda-4e06-90ae-07c90f105bbe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        }
      },
      "source": [
        "#example = next(iter(test_data))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-28-0aeb3d93aadf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mexample\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'test_data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYrHv1LNWr6b"
      },
      "source": [
        "#field = {'date':('date',TEXT),'text_split':('text_split',TEXT),'change_label':('change_label',LABEL)}\n",
        "#maybe i'll leave out the date first. \n",
        "field = {'text_split':('text_split',TEXT),'change_label':('change_label',LABEL)}\n",
        "#field = {'date':('date',TEXT), 'change_label':('change_label',LABEL)}\n",
        "#field = {'change_label':('change_label',LABEL)}\n",
        "\n",
        "#I need to do something about the 'text_split' field with this tokenizer, the processing might to be done before hand. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQIc-npDWr2c"
      },
      "source": [
        "train_1, test_1 = data.TabularDataset.splits(path='/content/gdrive/My Drive/Crypto/',train='df3_save.csv',test='df4_save.csv',format='csv',fields=field)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bGHigWuSX9K9",
        "outputId": "728a31c7-7c9e-47ef-c36c-aba775dae8d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#this step is apparently now needed if my sentence is not gazillion long. \n",
        "import sys\n",
        "import csv\n",
        "\n",
        "csv.field_size_limit(sys.maxsize)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "131072"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8rijb5PdY9oY",
        "outputId": "ffadd979-4edc-408d-b8d9-8704a5319ce1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print (train_1[0].__dict__.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['text_split', 'change_label'])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "as0V9IqVZV2l",
        "outputId": "dde46c50-aa70-43d4-be14-1e54fc907d08",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 59
        }
      },
      "source": [
        "print (train_1[0].__dict__.values())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_values([['btc', 'bitcoin', 'current', 'price', 'more', 'on', 'stick', 'to', 'bitcoin', 'script', 'during', 'ask', 'me', 'anything', 'chat', 'we', 'are', 'waiting', 'so', 'please', 'release', 'the', 'show', 'as', 'quick', 'as', 'possible', 'dinheiro', 'para', 'serve', 'se', 'dinheiro', 'responder', 'um', 'da', 'bitcoin', 'btc', 'crypto', 'see', 'mixed', 'bitcoin', 'still', 'above', 'con', 'en', 'panama', 'en', 'ya', 'bitcoin', 'bitcoin', 'give', 'two', 'about', 'this', 'at', 'least', 'also', 'launch', 'of', 'exchange', 'approval', 'of', 'bitcoin', 'if', 'those', 'two', 'occur', 'then', 'institutional', 'will', 'consider', 'cryptographic', 'currency', 'legitimate', 'which', 'believe', 'will', 'trigger', 'the', 'biggest', 'bull', 'run', 'in', 'financial', 'history', 'btc', 'bitcoin', 'wallet', 'forced', 'to', 'drop', 'key', 'privacy', 'from', 'play', 'is', 'still', 'live', 'do', 'not', 'miss', 'this', 'opportunity', 'of', 'get', 'the', 'best', 'price', 'for', 'days', 'before', 'the', 'private', 'sale', 'say', 'bitcoin', 'could', 'bankrupt', 'the', 'planet', 'climate', 'the', 'advocate', 'messenger', 'bitcoin', 'de', 'em', 'anterior', 'bitcoin', 'price', 'index', 'hong', 'based', 'physical', 'bitcoin', 'why', 'will', 'be', 'the', 'year', 'of', 'crypto', 'future', 'of', 'bitcoin', 'btc', 'eth', 'and', 'ripple', 'xrp', 'try', 'the', 'content', 'on', 'other', 'as', 'well', 'and', 'put', 'the', 'link', 'to', 'that', 'in', 'twitter', 'most', 'of', 'your', 'listener', 'follow', 'you', 'here', 'also', 'as', 'follower', 'would', 'rather', 'send', 'my', 'your', 'link', 'on', 'medium', 'eventually', 'critical', 'mass', 'will', 'be', 'there', 'like', 'bitcoin', 'si', 'ce', 'does', 'not', 'have', 'monetary', 'policy', 'it', 'is'], '1'])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KN4lfcSUZaRI",
        "outputId": "aca75a56-cbb0-42a1-e1c1-653710bbdc4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        }
      },
      "source": [
        "next(iter(train_1)).change_label\n",
        "#so the text_split part, is a list. of al the tokens i recon, and '[' is even tokenized as well. \n",
        "#now it looks somewhat normal. "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bKpu2s7kPH2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3EJPZUSZgTl",
        "outputId": "bde2a5a8-4950-484b-ccea-6a0e38aedf2c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "#now let's try out the build vocabulary part\n",
        "\n",
        "MAX_VOCAB_SIZE = 25_000\n",
        "\n",
        "TEXT.build_vocab(train_1, \n",
        "                 max_size = MAX_VOCAB_SIZE, \n",
        "                 vectors = \"glove.6B.100d\", \n",
        "                 unk_init = torch.Tensor.normal_) # how to initialize unseen words not in glove\n",
        "\n",
        "LABEL.build_vocab(train_1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".vector_cache/glove.6B.zip: 862MB [06:30, 2.21MB/s]                           \n",
            "100%|█████████▉| 399433/400000 [00:18<00:00, 21857.88it/s]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cXINOdTToVmb",
        "outputId": "2f299db0-0996-42ea-a709-1ed2a4d9bfb9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        }
      },
      "source": [
        "TEXT.pad_token"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<pad>'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "teseLF-jNJGM"
      },
      "source": [
        "#so...now that I've built the vocab, how do I check? obviously this vocab will not be very crypto currency friendly. \n",
        "#so ideally it should also be fine tuned. but let's get on with it for now. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hV7PZYVIPYPj"
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRmB3WIaRsSW"
      },
      "source": [
        "train_iterator= data.BucketIterator(\n",
        "    train_1, \n",
        "    batch_size = BATCH_SIZE,\n",
        "    #sort_within_batch = True,\n",
        "    sort_key=lambda x: len(x.text_split), # the BucketIterator needs to be told what function it should use to group the data.\n",
        "    sort_within_batch=False,\n",
        "    device = device)\n",
        "\n",
        "#error msg: Example' object has no attribute 'sort_key'\n",
        "#stackoverflow says fields are not passed in the same order as they are in the csv/tsv file\n",
        "#tried to convert labels from true/false to 1/0, didn't solve the error. \n",
        "#check values, nothing is null. how to solve this buuuuug. HALP!!!\n",
        "\n",
        "#...ok, I didn't need to split, because, i'm only iterating over train_l data...ok work! "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2NIv5U4Z4Ci9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pVGrJXKh4DFu"
      },
      "source": [
        "# Get into the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lNPRtNEyR3ul"
      },
      "source": [
        "##now, let's get into the modeling part \n",
        "#1. defining the model\n",
        "#2. writting out the training loop \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uatG5C5GWSvY"
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class RNN(nn.Module):\n",
        "    def __init__(self,vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
        "                 bidirectional, dropout, pad_idx):\n",
        "      super().__init__()\n",
        "      self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)\n",
        "      self.rnn = nn.LSTM(embedding_dim,\n",
        "                         hidden_dim,\n",
        "                         num_layers=n_layers,\n",
        "                         bidirectional=bidirectional,\n",
        "                         dropout=dropout)\n",
        "      self.fc = nn.Linear(hidden_dim*2, output_dim)\n",
        "      self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, text, text_lengths):\n",
        "      embedded = self.dropout(self.embedding(text))\n",
        "      \n",
        "      packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths,enforce_sorted=False) #added the enforced_sorted = false, debug\n",
        "      packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
        "\n",
        "      output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)#btw this is not used here i think.\n",
        "\n",
        "      hidden = self.dropout(torch.cat((hidden[-2,:,:],hidden [-1,:,:]),dim =1 ))\n",
        "\n",
        "      return self.fc(hidden)\n",
        "\n",
        "\n",
        "      "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sW4Nx_h6WTuX"
      },
      "source": [
        "INPUT_DIM=len(TEXT.vocab)\n",
        "EMBEDDING_DIM=100\n",
        "HIDDEN_DIM=256\n",
        "OUTPUT_DIM = 1\n",
        "N_LAYERS=2\n",
        "BIDIRECTIONAL=True\n",
        "DROPOUT = 0.5\n",
        "PAD_IDX= TEXT.vocab.stoi[TEXT.pad_token]\n",
        "\n",
        "#i really donno what's the pad_token thing here...or how to use/where to learn vocab.stoi \n",
        "#TEXT.pad_token is just the dictionary key to the padding token, which is <pad> here.\n",
        "#and TEXT.vocab.stoi['<pad>'] gives the padding value, 1. \n",
        "model = RNN(INPUT_DIM,EMBEDDING_DIM,HIDDEN_DIM,OUTPUT_DIM,N_LAYERS,BIDIRECTIONAL,DROPOUT,PAD_IDX)\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aran0GdKpETq",
        "outputId": "8af678d4-df60-4b02-cdac-73f615003ae6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#checking trainable parameters\n",
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 3,257,857 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WHPHSzWpOTW",
        "outputId": "d9ec89ef-2e60-43ea-d2f0-f940be2571eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "\n",
        "print(pretrained_embeddings.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([9472, 100])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wPGK0EJpmfj",
        "outputId": "22ad6fa5-707f-4db1-815d-2124c58ab944",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "#copying the pretrained weights into the embedding. \n",
        "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
        "\n",
        "\n",
        "#i'm a little bit confused about this part for sure..why copying weights? then the embeddings is what? \n",
        "#so embedding layer is just like a dictionary with word index and as the key, values are just hte \"weights\"\n",
        "#if i dont'wnat the weigts to be touched, I can specify in the model as\n",
        "# print(\"Do not finetune word embedding layer.\")\n",
        "#             self.emb.weight.requires_grad = False\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.1117, -0.4966,  0.1631,  ...,  1.2647, -0.2753, -0.1325],\n",
              "        [-0.8555, -0.7208,  1.3755,  ...,  0.0825, -1.1314,  0.3997],\n",
              "        [ 0.8403,  0.1327, -0.6158,  ...,  0.7602, -0.2352,  0.3349],\n",
              "        ...,\n",
              "        [ 0.7893,  0.6285, -0.4647,  ...,  0.0823,  0.8793, -0.0854],\n",
              "        [ 0.0907,  0.9267, -0.4509,  ...,  0.0772, -0.1289,  0.1862],\n",
              "        [ 0.1432,  0.0585, -0.9011,  ...,  0.0697, -0.1663,  0.2665]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POwW4z-kpptF",
        "outputId": "e367ec8f-d6d9-4e32-99ae-66e1465da5b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "#this step will set the <unk> and <pad> to be 0, and make them irrelevant for the model\n",
        "#is this good though? the pretrained model is not on crypto words, which means the model WILL ignore crypto words\n",
        "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "\n",
        "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
        "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
        "\n",
        "print(model.embedding.weight.data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
            "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
            "        [ 0.8403,  0.1327, -0.6158,  ...,  0.7602, -0.2352,  0.3349],\n",
            "        ...,\n",
            "        [ 0.7893,  0.6285, -0.4647,  ...,  0.0823,  0.8793, -0.0854],\n",
            "        [ 0.0907,  0.9267, -0.4509,  ...,  0.0772, -0.1289,  0.1862],\n",
            "        [ 0.1432,  0.0585, -0.9011,  ...,  0.0697, -0.1663,  0.2665]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5gOwUwHhNTW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xxWZgTiuhQnS"
      },
      "source": [
        "# Now Train the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ROIJ_wZahSzf"
      },
      "source": [
        "#choose optimizer \n",
        "import torch.optim as optim\n",
        "\n",
        "optimizer = optim.Adam(model.parameters())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYgk02YEixY8"
      },
      "source": [
        "#moving to GPU. \n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "\n",
        "model= model.to(device)\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y0v_m0UVjKo2"
      },
      "source": [
        "#defining accuracy metrics\n",
        "def binary_accuracy(preds, y):\n",
        "    \"\"\"\n",
        "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
        "    \"\"\"\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ibwmgjtblNSw"
      },
      "source": [
        "#note, here we seperate the text and text.length as they are tuple\n",
        "\n",
        "def train(model,iterator,optimizer, criterion):\n",
        "  epoch_loss = 0 \n",
        "  epoch_acc = 0\n",
        "\n",
        "  model.train() #so that droput function works \n",
        "  for batch in iterator: \n",
        "    optimizer.zero_grad()#why? ok, to not accumulate un-intended gradient i guess.\n",
        "\n",
        "    text, text_lengths = batch.text_split\n",
        "    predictions = model(text, text_lengths).squeeze(1) #what's the squeeze, what does squeeze do? pack np.array to be lower dimension?\n",
        "    loss = criterion(predictions, batch.change_label)\n",
        "\n",
        "    acc = binary_accuracy(predictions, batch.change_label)\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    epoch_loss += loss.item()\n",
        "    epoch_acc += acc.item()\n",
        "\n",
        "    return epoch_loss / len(iterator) , epoch_acc / len(iterator)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sFljptalqM38"
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "\n",
        "  epoch_loss = 0 \n",
        "  epoch_acc = 0\n",
        "\n",
        "  model.evaluate() #so that droput function DONT ACTIVATE\n",
        "\n",
        "  with torch.no_grad():\n",
        "  #is this what we do for evaluation always? \n",
        "  #answer: \"with torch.no_grad()\" temporarily set all the requires_grad flag to false.\n",
        "    for batch in iterator: \n",
        "      text, text_lengths = batch.text_split\n",
        "      predictions = model(text, text_lengths).squeeze(1) #what's the squeeze, what does squeeze do? pack np.array to be lower dimension?\n",
        "      loss = criterion(predictions, batch.change_label)\n",
        "\n",
        "      acc = binary_accuracy(predictions, batch.change_label)\n",
        "\n",
        "      epoch_loss += loss.item()\n",
        "      epoch_acc += acc.item()\n",
        "\n",
        "  return epoch_loss / len(iterator) , epoch_acc / len(iterator)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5KmMLzLq9PH"
      },
      "source": [
        "#define a timing function \n",
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Y_xyJhlsFcU",
        "outputId": "831a9307-bcc6-44c2-ad4a-6b2e4fc57d16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "#now try to train yea\n",
        "N_EPOCHS = 5\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    #start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "    #valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "    \n",
        "    #end_time = time.time()\n",
        "\n",
        "    #epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    # if valid_loss < best_valid_loss:\n",
        "    #     best_valid_loss = valid_loss\n",
        "    #     torch.save(model.state_dict(), 'tut2-model.pt')\n",
        "    \n",
        "    #print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    #print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\tTrain Loss: 0.016 | Train Acc: 1.31%\n",
            "\tTrain Loss: 0.016 | Train Acc: 1.31%\n",
            "\tTrain Loss: 0.016 | Train Acc: 1.09%\n",
            "\tTrain Loss: 0.016 | Train Acc: 1.34%\n",
            "\tTrain Loss: 0.016 | Train Acc: 1.38%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bo1cq21ZscpO"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}