data-preprocessing-tweets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "import matplotlib.pyplot as plt\n",
    "importlib.reload(dp)\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.kaggle.com/kazanova/sentiment140/data\n",
    "file_path1 = \"./data/training.1600000.processed.noemoticon.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1577443\n",
      "[\"@hodgman but she's deeeaaad  - a batman's opinion\", \"@oconel me too - she usually is early and i think her computer was on when i got here but now it looks like it's locked or she's left \"]\n"
     ]
    }
   ],
   "source": [
    "tweets = list(set(dp.load_data(file_path1, header=False, sep=',', usecols=[5])[5].str.lower().values))\n",
    "print(len(tweets))\n",
    "print(tweets[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150\n",
      "['\\t', ' ', '!', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '=', '?', '@', '[', '\\\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '\\x7f', '\\xa0', '¡', '¢', '£', '¤', '¥', '¦', '§', '¨', '©', 'ª', '«', '¬', '\\xad', '®', '¯', '°', '±', '²', '³', '´', 'µ', '¶', '·', '¸', '¹', 'º', '»', '¼', '½', '¾', '¿', '×', 'ß', 'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 'ð', 'ñ', 'ò', 'ó', 'ø', 'ù', 'ú', 'û', 'ÿ', 'œ', 'š', 'ž', 'ƒ', 'ˆ', '˜', '–', '—', '‘', '’', '‚', '“', '”', '„', '†', '‡', '•', '…', '‰', '‹', '›', '€', '™']\n"
     ]
    }
   ],
   "source": [
    "unique_chars = dp.get_unique_chars(tweets)\n",
    "print(len(unique_chars))\n",
    "print(unique_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dp.get_char_presence(quotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of tweets =  1577443\n",
      "577841\n",
      "['seriously, why is it so difficult to find a decide lamp shade around here?  two days of looking and still no winners ', \"morning all. it's raining here  hope the gals had a ball last night - duh i reckon that's a given xx\", 'uhh! surveys! dlsu-m people! who wants to answer surveys? pm me ', 'loving rich people who flaunt their money ', 'is looking at some old pictures.. ah!! those were the days.. ', 'dont wanna go back to lex, where only sadness and pain awaits me ', 'i feel bad, this headache is very strong!!!  sometimes it bothers to be sick...', 'on a new work out routine  so pooped after it!', 'is at south molton vintage ralley! ', 'going shopping right now ']\n"
     ]
    }
   ],
   "source": [
    "print(\"Total number of tweets = \", len(tweets))\n",
    "filter_tweets = dp.filter_data(tweets, \"^[a-zA-Z !\\\"',.?\\-\\:\\*]+$\")\n",
    "print(len(filter_tweets))\n",
    "print(filter_tweets[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n",
      "[' ', '!', \"'\", '*', ',', '-', '.', ':', '?', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
      "577841\n",
      "[['seriously', ',', 'why', 'is', 'it', 'so', 'difficult', 'to', 'find', 'a', 'decide', 'lamp', 'shade', 'around', 'here', '?', 'two', 'days', 'of', 'looking', 'and', 'still', 'no', 'winners'], ['morning', 'all', '.', \"it's\", 'raining', 'here', 'hope', 'the', 'gals', 'had', 'a', 'ball', 'last', 'night', '-', 'duh', 'i', 'reckon', \"that's\", 'a', 'given', 'xx']]\n",
      "min of length:  1\n",
      "max of length:  65\n",
      "average length:  13.406289965578766\n",
      "mode of length:  7\n"
     ]
    }
   ],
   "source": [
    "pad_chars = [' ', '!', '\"', '*', ',', '-', ':', '?']\n",
    "clean_tweets = dp.clean_data(tweets, \"^[a-zA-Z !\\\"'*,\\-.:?]+$\", pad_chars)\n",
    "\n",
    "unique_chars = dp.get_unique_chars(clean_tweets)\n",
    "print(len(unique_chars))\n",
    "print(unique_chars)\n",
    "\n",
    "clean_tweets = dp.tokenize(clean_tweets)\n",
    "\n",
    "print(len(clean_tweets))\n",
    "print(clean_tweets[:2])\n",
    "\n",
    "# stats of number of tokens in the sentence\n",
    "dp.get_len_stats(clean_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "362932\n"
     ]
    }
   ],
   "source": [
    "MIN_NUM_TOKENS = 10\n",
    "MAX_NUM_TOKENS = 40\n",
    "clean_tweets = dp.filter_data_on_length(clean_tweets, MIN_NUM_TOKENS, MAX_NUM_TOKENS)\n",
    "print(len(clean_tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freqs = dp.get_vocabulary(clean_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total vocabulary:  120011\n",
      "average of frequency:  53.44429260651107\n",
      "mode of frequency:  1\n"
     ]
    }
   ],
   "source": [
    "dp.get_vocabulary_stats(word_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of filtered words =  10146\n",
      "192898\n",
      "[['seriously', ',', 'why', 'is', 'it', 'so', 'difficult', 'to', 'find', 'a', 'decide', 'lamp', 'shade', 'around', 'here', '?', 'two', 'days', 'of', 'looking', 'and', 'still', 'no', 'winners'], ['morning', 'all', '.', \"it's\", 'raining', 'here', 'hope', 'the', 'gals', 'had', 'a', 'ball', 'last', 'night', '-', 'duh', 'i', 'reckon', \"that's\", 'a', 'given', 'xx'], ['is', 'looking', 'at', 'some', 'old', 'pictures', 'threedots', 'ah', '!', 'those', 'were', 'the', 'days', 'threedots'], ['i', 'feel', 'bad', ',', 'this', 'headache', 'is', 'very', 'strong', '!', 'sometimes', 'it', 'bothers', 'to', 'be', 'sick', 'threedots'], ['on', 'a', 'new', 'work', 'out', 'routine', 'so', 'pooped', 'after', 'it', '!']]\n"
     ]
    }
   ],
   "source": [
    "filtered_words = dp.filter_words(word_freqs, 20)\n",
    "print(\"Number of filtered words = \", len(filtered_words))\n",
    "sents = dp.get_sentences_with_words(clean_tweets, filtered_words)\n",
    "print(len(sents))\n",
    "print(sents[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHoVJREFUeJzt3Xl8VfWd//HXhwQCyBJ2wibIIoIKSorUblarotWhfUyt2I12nHFqtdNOl1Hb+Y2dLjN17L7ZOtUWl5ZaWkdkdKhrq48KElTCEpHIIjEBAmGnJCT5/P6439hLvElOQm7OXd7Px+M+cu/3fs85329OknfuOed+rrk7IiIiUfSKewAiIpI9FBoiIhKZQkNERCJTaIiISGQKDRERiUyhISIikSk0REQkMoWGiIhEptAQEZHICuMeQHcbPny4T5w4Me5hiIhklTVr1uxx9xEd9cu50Jg4cSJlZWVxD0NEJKuY2fYo/XR4SkREIlNoiIhIZAoNERGJTKEhIiKRKTRERCQyhYaIiESm0BARkcgUGiIiWczdWVd1gP98pII9h+vTvr2ce3OfiEg+2LTzEA+vrWZ5eTXb9h6lsJcxd9JQLjpjVFq3q9AQEckSr9YeZvnaGpaXV7N592F6GZw/eTjXXzCZS2eOprh/n7SPQaEhIpLBdtQd5eHyapavrWFjzUHM4C0Th/K1953JZWeOZviAoh4dj0JDRCTD1Bz4C/9bXsPD5TWs3bEfgHMmFPP/rpjBe88qYfTgvrGNTaEhIpIBag/V8+j6GpavreH5bXUAnDl2EDdfNp33nlXC+KH9Yx5hgkJDRCQm+4408H8bdrK8vJrnXt1Ls8Ppowby+YunccWsMUwafkrcQ3wThYaISA86eOw4f9iwi+Xl1Ty7eQ+Nzc6k4adw47uncMWsMUwbNTDuIbarw9Aws77An4Ci0H+pu99qZr8E3gUcCF0/7u4vmZkB3wcuB46G9hfCuhYB/xr6f93dF4f2OcAvgX7AI8Bn3N3NbCjwG2AisA34oLvvO8k5i4j0qKMNjTxesZuH11bzx021NDQ1M7a4H9e+YxJXnj2GmWMGkfjTmfmivNKoBy5098Nm1ht41sweDc990d2Xtup/GTA13M4D7gDOCwFwK1AKOLDGzJaFELgDuA5YSSI05gOPAjcDT7j7N83s5vD4pq5PV0SkZxw73sTTm3bzcHkNT1Ts4tjxZkYNKuIj807lylklzB5fnDVBkazD0HB3Bw6Hh73DzdtZZAFwT1hupZkVm1kJcAHwmLvXAZjZY8B8M3saGOTuz4X2e4D3kQiNBWE5gMXA0yg0RCRDNTQ288zmWpaX1/CHDTs50tDE8AF9uGrOeK44u4S3TBxKr17ZFxTJIp3TMLMCYA0wBfixu68ys+uBb5jZvwFPADe7ez0wFtiRtHhVaGuvvSpFO8Aod68BcPcaMxvZyfmJiKRVY1Mzf351L8vLq/m/9Ts5eKyRwf16c+WsMVxx9hjmnTaUwoLcqdgUKTTcvQmYbWbFwINmdiZwC7AT6APcSeIVwFeBVDHqXWiPzMyuI3F4iwkTJnRmURGRTmtqdlZvq+PhtYmg2HukgQFFhVwyYxRXzhrD26YMp09h7gRFsk5dPeXu+8PhpPnu/q3QXG9mvwC+EB5XAeOTFhsHVIf2C1q1Px3ax6XoD7DLzErCq4wSYHcb47qTRHBRWlraqcAREYnC3Xnhtf08vLaaR9bVsPtQPf16F3DRGSO5ctYY3jVtBH17F8Q9zLSLcvXUCOB4CIx+wHuA25L+mBuJcxDrwyLLgBvNbAmJE+EHQr8VwH+Y2ZDQ7xLgFnevM7NDZjYPWAV8DPhh0roWAd8MXx/qjkmLiETh7qx//SDLy6tZXl7D6/v/Qp/CXrz79BFccfYYLjpjJP375Nc7F6LMtgRYHM5r9AIecPflZvZkCBQDXgI+Gfo/QuJy20oSl9x+AiCEw9eA1aHfV1tOigPX89dLbh8NN0iExQNmdi3wGnBVVycqIhJVqgqy75w2gs9fMo2LZ4xiYN/ecQ8xNpa4yCl3lJaWellZWdzDEJEs01YF2StnlfRYBdk4mdkady/tqF9+va4SEUmSaRVks4FCQ0TySiZXkM0GCg0RyXktFWQfXlvN6m2JSkSZWEE2Gyg0RCQnZWMF2Wyg0BCRnJHtFWSzgUJDRLLakfpGHq/YxfLymqyvIJsNFBoiknXeqCC7toYnXs6dCrLZQKEhIlkhHyrIZgOFhohkrHyrIJsNFBoiklHyuYJsNlBoiEjsVEE2eyg0RCQWqiCbnbRHRKTHuDubdh16ozCgKshmH4WGiKRdWxVkr79gcl5UkM0lCg0RSQtVkM1NCg0R6TaqIJv7FBoiclJUQTa/KDREpNNUQTZ/KTREJBJVkBVQaIhIO1RBVlpTaIjICVRBVtqj0BCRlBVkh52iCrLyZgoNkTylCrLSFQoNkTzS1Ow8v7XujaBQBVnpLIWGSI5TBVnpTgoNkRykCrKSLvqpEckRqiArPUGhIZLlVEFWepJCQyQLqYKsxEWhIZIlVEFWMoFCQySDqYKsZJoOQ8PM+gJ/AopC/6XufquZTQKWAEOBF4CPunuDmRUB9wBzgL3A1e6+LazrFuBaoAn4J3dfEdrnA98HCoCfu/s3Q3vKbXTT3EUykirISiaL8kqjHrjQ3Q+bWW/gWTN7FPgc8F13X2JmPyURBneEr/vcfYqZLQRuA642sxnAQmAmMAZ43MymhW38GLgYqAJWm9kyd98Ylk21DZGcogqyki06DA13d+BweNg73By4EPhQaF8MfIXEH/QF4T7AUuBHlqhutgBY4u71wFYzqwTmhn6V7r4FwMyWAAvMrKKdbYhkPVWQlWwU6ZyGmRUAa4ApJF4VvArsd/fG0KUKGBvujwV2ALh7o5kdAIaF9pVJq01eZker9vPCMm1tQyQrqYKsZLtIoeHuTcBsMysGHgTOSNUtfE31E+/ttKcqdNNe/zcxs+uA6wAmTJiQqotIbFRBVnJJp66ecvf9ZvY0MA8oNrPC8EpgHFAdulUB44EqMysEBgN1Se0tkpdJ1b6nnW20HtedwJ0ApaWlKYNFpCepgqzkqihXT40AjofA6Ae8h8QJ6qeAD5C4umkR8FBYZFl4/Fx4/kl3dzNbBvzKzL5D4kT4VOB5Eq8opoYrpV4ncbL8Q2GZtrYhknFUQVbyQZRXGiXA4nBeoxfwgLsvN7ONwBIz+zrwInBX6H8XcG840V1HIgRw9w1m9gCwEWgEbgiHvTCzG4EVJC65vdvdN4R13dTGNkQygirISr6xxMVRuaO0tNTLysriHobkMFWQlVxkZmvcvbSjfvrJFolAFWRFEhQaIu1QBVmREyk0RFpRBVmRtik0RFAFWZGoFBqSt1RBVqTzFBqSV1JVkJ02aoAqyIpEpNCQnNdWBdkb3j2FK84ew+mjVUFWJCqFhuQkVZAVSQ+FhuQMVZAVST+FhmQ1VZAV6VkKDck6qiArEh+FhmQFVZAVyQwKDclYqiArknkUGpJRVEFWJLPpt09ipwqyItlDoSGxUQVZkeyj0JAepQqyItlNoSFppwqyIrlDoSFpoQqyIrlJoSHdpqWC7MNrq1m5RRVkRXKRQkNOiirIiuQXhYZ0mirIiuQvhYZEogqyIgIKDWmHKsiKSGsKDTmBKsiKSHsUGqIKsiISmUIjT6mCrIh0hUIjj6iCrIicLP2FyHGqICsi3UmhkaNUQVZE0kGhkUNUQVZE0q3D0DCz8cA9wGigGbjT3b9vZl8B/gGoDV2/5O6PhGVuAa4FmoB/cvcVoX0+8H2gAPi5u38ztE8ClgBDgReAj7p7g5kVhW3PAfYCV7v7tm6Yd86ob2zi/pWv8dDaalWQFZG0i/JKoxH4vLu/YGYDgTVm9lh47rvu/q3kzmY2A1gIzATGAI+b2bTw9I+Bi4EqYLWZLXP3jcBtYV1LzOynJALnjvB1n7tPMbOFod/VJzPhXHPrQxtYsnqHKsiKSI/oMDTcvQaoCfcPmVkFMLadRRYAS9y9HthqZpXA3PBcpbtvATCzJcCCsL4LgQ+FPouBr5AIjQXhPsBS4EdmZu7ukWeYw5Y8/xpLVu/ghndP5ouXTo97OCKSBzr1ji0zmwicA6wKTTeaWbmZ3W1mQ0LbWGBH0mJVoa2t9mHAfndvbNV+wrrC8wdC/7xXXrWff1u2gXdMHc7nLj497uGISJ6IHBpmNgD4HfBZdz9I4pXAZGA2iVci327pmmJx70J7e+tqPbbrzKzMzMpqa2tTLJJb6o40cP19LzBiQBHfX3gOBar/JCI9JFJomFlvEoFxv7v/HsDdd7l7k7s3A//NXw9BVQHjkxYfB1S3074HKDazwlbtJ6wrPD8YqGs9Pne/091L3b10xIgRUaaUtZqanc8seZHaQ/Xc8ZFzGXqKLp0VkZ7TYWhYot71XUCFu38nqb0kqdv7gfXh/jJgoZkVhauipgLPA6uBqWY2ycz6kDhZviycn3gK+EBYfhHwUNK6FoX7HwCezPfzGd97/BWe2byHry6YydnjiuMejojkmShXT70N+CiwzsxeCm1fAq4xs9kkDhdtA/4RwN03mNkDwEYSV17d4O5NAGZ2I7CCxCW3d7v7hrC+m4AlZvZ14EUSIUX4em84mV5HImjy1uMbd/HDJyu5unQ8C+dOiHs4IpKHLNf+cS8tLfWysrK4h9Httu05wpU/epZTh/Vn6SfPVzFBEelWZrbG3Us76qd611ngLw1NfPK+NRT0Mu748BwFhojERmVEMpy786UH17Fp1yF+8fG36I17IhIrvdLIcPeu3M6DL77OP79nGhecPjLu4YhInlNoZLA12/fxteUbuXD6SG5895S4hyMiotDIVLWH6vnU/WsoGdyP735wNr30Bj4RyQA6p5GBGpua+fSvX2D/0eM8+Km5DO6vD0kSkcyg0MhAt6/YxMotdXz7qlnMGDMo7uGIiLxBh6cyzKPravjZn7bw0Xmn8rdzxsU9HBGREyg0Mkjl7sN84bdr3/gQJRGRTKPQyBCH6xv55H1r6Nu7gJ98+Fz6FGrXiEjm0TmNDODu3LS0nC21h7nv78+jZHC/uIckIpKS/p3NAHc9u5X/XVfDTfOnc/7k4XEPR0SkTQqNmK3cspf/fPRl5s8czXXvPC3u4YiItEuhEaOdB45x469e4NRh/bn9qrNJfHSJiEjmUmjEpKGxmRt+9QJHG5r42UfmMLCv3sAnIplPJ8Jj8h+PVLBm+z5+eM05TB01MO7hiIhEolcaMVj/+gF++edtfPz8iVw5a0zcwxERiUyhEYPbV2yiuH9vPnfJtLiHIiLSKQqNHrZqy17++Eot179rMoN0HkNEsoxCowe5O7ev2MSoQUUsOn9i3MMREek0hUYPemrTbsq27+PTF07V53yLSFZSaPSQ5mbn9hWvcOqw/lz9lvFxD0dEpEsUGj1k+boaKmoO8rmLp9G7QN92EclO+uvVA443NfOdP2xi+uiBXHm2LrEVkeyl0OgBvy2rYtveo3zhktP1Wd8iktUUGml27HgTP3hiM+dOKOaiM0bGPRwRkZOi0Eize5/bzs6Dx/jipdNVkFBEsp5CI40OHTvOT56u5B1Th/PWycPiHo6IyElTaKTRz5/Zyr6jx/mXS6fHPRQRkW6h0EiTvYfr+fkzW7jszNGcNW5w3MMREekWCo00+cnTr/KX4018XkUJRSSHKDTSYP/RBu5buZ33nzOOKSP1WRkikjs6DA0zG29mT5lZhZltMLPPhPahZvaYmW0OX4eEdjOzH5hZpZmVm9m5SetaFPpvNrNFSe1zzGxdWOYHFi4zamsbmW7pmirqG5u59u2T4h6KiEi3ivJKoxH4vLufAcwDbjCzGcDNwBPuPhV4IjwGuAyYGm7XAXdAIgCAW4HzgLnArUkhcEfo27Lc/NDe1jYyVnOzc/+q15hz6hBmjBkU93BERLpVh6Hh7jXu/kK4fwioAMYCC4DFodti4H3h/gLgHk9YCRSbWQlwKfCYu9e5+z7gMWB+eG6Quz/n7g7c02pdqbaRsf786l627jnCR+ZNiHsoIiLdrlPnNMxsInAOsAoY5e41kAgWoOXtzmOBHUmLVYW29tqrUrTTzjZaj+s6Myszs7La2trOTKnb3bdyO0P69+ayM0tiHYeISDpEDg0zGwD8Dvisux9sr2uKNu9Ce2Tufqe7l7p76YgRIzqzaLfaeeAYj1Xs4oOl4/V5GSKSkyKFhpn1JhEY97v770PzrnBoifB1d2ivApI/MGIcUN1B+7gU7e1tIyMtWf0aze586DwdmhKR3BTl6ikD7gIq3P07SU8tA1qugFoEPJTU/rFwFdU84EA4tLQCuMTMhoQT4JcAK8Jzh8xsXtjWx1qtK9U2Ms7xpmZ+/fxrvHPqCE4ddkrcwxERSYvCCH3eBnwUWGdmL4W2LwHfBB4ws2uB14CrwnOPAJcDlcBR4BMA7l5nZl8DVod+X3X3unD/euCXQD/g0XCjnW1knCcqdrHrYD1ff9+pcQ9FRCRtOgwNd3+W1OcdAC5K0d+BG9pY193A3Snay4AzU7TvTbWNTHTfytcYM7gvF05X+XMRyV16R3g32LrnCM9W7uGauRMo0IcsiUgOU2h0g/tXbqewl3H13PEddxYRyWIKjZPU2NTM/7z0OhfPGMXIgX3jHo6ISFopNE7Sn1/dy57DDSyYPbbjziIiWU6hcZKWra1mYFEhF5we35sKRUR6ikLjJBw73sSK9Tu59MzRege4iOQFhcZJeHpTLYfqG/mbWWPiHoqISI9QaJyEh9dWM3xAH86fPCzuoYiI9AiFRhcdrm/k8YpdXH5WCYUF+jaKSH7QX7su+sOGndQ3NuvQlIjkFYVGFy1bW83Y4n6cOyErPoFWRKRbKDS6YP/RBp7dvIcrZpXQS2VDRCSPKDS64OlNtTQ2O/Nnjo57KCIiPUqh0QWPV+xi+IAiZo0rjnsoIiI9SqHRSQ2NzfxxUy0XTR+pQ1MikncUGp20elsdh+obec+MUXEPRUSkxyk0OumxjbsoKuzF26cMj3soIiI9TqHRCe7O4xW7ePuU4fTro1pTIpJ/FBqd8Mquw1Tt+4sOTYlI3lJodMKTL+8G0OeAi0jeUmh0wjOba5k+eiCjBukT+kQkPyk0Ijra0EjZtn28c5o+bElE8pdCI6JVW+toaGrmHVN11ZSI5C+FRkTPvLKHPoW9eMvEoXEPRUQkNgqNiJ6trOW8SUP1sa4iktcUGhHsPHCMV3Yd1qEpEcl7Co0IntuyB4DzJys0RCS/KTQiWPlqHYP6FnJGyaC4hyIiEiuFRgSrtu5l7qRhFKiqrYjkOYVGB3YeOMa2vUeZd5qumhIRUWh0YNXWvQDMO21YzCMREYlfh6FhZneb2W4zW5/U9hUze93MXgq3y5Oeu8XMKs1sk5ldmtQ+P7RVmtnNSe2TzGyVmW02s9+YWZ/QXhQeV4bnJ3bXpDtj5Za9DCzS+QwREYj2SuOXwPwU7d9199nh9giAmc0AFgIzwzI/MbMCMysAfgxcBswArgl9AW4L65oK7AOuDe3XAvvcfQrw3dCvx63eto85E4fofIaICBFCw93/BNRFXN8CYIm717v7VqASmBtule6+xd0bgCXAAjMz4EJgaVh+MfC+pHUtDveXAheF/j1m35EGKncf1rvARUSCkzmncaOZlYfDV0NC21hgR1KfqtDWVvswYL+7N7ZqP2Fd4fkDof+bmNl1ZlZmZmW1tbUnMaUTrdm+D4DSU4d00FNEJD90NTTuACYDs4Ea4NuhPdUrAe9Ce3vrenOj+53uXurupSNGdF8V2tXb6+hdYMwaX9xt6xQRyWZdCg133+XuTe7eDPw3icNPkHilMD6p6zigup32PUCxmRW2aj9hXeH5wUQ/TNYt1mzbx5ljB6velIhI0KXQMLOSpIfvB1qurFoGLAxXPk0CpgLPA6uBqeFKqT4kTpYvc3cHngI+EJZfBDyUtK5F4f4HgCdD/x5R39hEedUBHZoSEUlS2FEHM/s1cAEw3MyqgFuBC8xsNonDRduAfwRw9w1m9gCwEWgEbnD3prCeG4EVQAFwt7tvCJu4CVhiZl8HXgTuCu13AfeaWSWJVxgLT3q2nbCx+iANTc2cO0GhISLSosPQcPdrUjTflaKtpf83gG+kaH8EeCRF+xb+engruf0YcFVH40uXl3bsB+AchYaIyBv0jvA2vLRjP6MH9WX0YH0euIhIC4VGG17asZ/ZumpKROQECo0U9h1pYPveo8yeoNAQEUmm0EhhbVXifMbZ4wbHPBIRkcyi0EihvOoAZnDWWIWGiEgyhUYKa3fs57ThpzCwb++4hyIiklEUGq24O2urDjBrnM5niIi0ptBoZfehevYcrucsnc8QEXkThUYr66oOAHCmzmeIiLyJQqOV9dWJk+Az9El9IiJvotBoZf3rBzlt+CmcUtRhhRURkbyj0GhlY/UBZo7RoSkRkVQUGkn2HWmg+sAxZo7RoSkRkVQUGkkqag4C6JWGiEgbFBpJNobQOKNkYMwjERHJTAqNJBU1hxg5sIhhA4riHoqISEZSaCSpqDnIdF1qKyLSJoVGcLypmcrdhzljtA5NiYi0RaERbN1zhIamZqbrfIaISJsUGsHLOw8BMH20Dk+JiLRFoRG8XHOQwl7G5BED4h6KiEjGUmgEE4b252/PHUefQn1LRETaogJLwcK5E1g4d0LcwxARyWj6t1pERCJTaIiISGQKDRERiUyhISIikSk0REQkMoWGiIhEptAQEZHIFBoiIhKZuXvcY+hWZlYLbE9qGg7siWk46ZJrc8q1+UDuzSnX5gOaU2unuvuIjjrlXGi0ZmZl7l4a9zi6U67NKdfmA7k3p1ybD2hOXaXDUyIiEplCQ0REIsuH0Lgz7gGkQa7NKdfmA7k3p1ybD2hOXZLz5zRERKT75MMrDRER6SY5HRpmNt/MNplZpZndHPd4kpnZeDN7yswqzGyDmX0mtA81s8fMbHP4OiS0m5n9IMyl3MzOTVrXotB/s5ktSmqfY2brwjI/MDPrgXkVmNmLZrY8PJ5kZqvC2H5jZn1Ce1F4XBmen5i0jltC+yYzuzSpvcf3p5kVm9lSM3s57Ku3ZvM+MrN/Dj9v683s12bWN9v2kZndbWa7zWx9Ulva90lb20jjnG4PP3flZvagmRUnPdep739X9nGb3D0nb0AB8CpwGtAHWAvMiHtcSeMrAc4N9wcCrwAzgP8Cbg7tNwO3hfuXA48CBswDVoX2ocCW8HVIuD8kPPc88NawzKPAZT0wr88BvwKWh8cPAAvD/Z8C14f7nwJ+Gu4vBH4T7s8I+6oImBT2YUFc+xNYDPx9uN8HKM7WfQSMBbYC/ZL2zcezbR8B7wTOBdYntaV9n7S1jTTO6RKgMNy/LWlOnf7+d3YftzvWdP/SxXULO31F0uNbgFviHlc7430IuBjYBJSEthJgU7j/M+CapP6bwvPXAD9Lav9ZaCsBXk5qP6FfmuYwDngCuBBYHn7p9iT94L+xT4AVwFvD/cLQz1rvp5Z+cexPYBCJP7LWqj0r9xGJ0NhB4g9lYdhHl2bjPgImcuIf2LTvk7a2ka45tXru/cD9qb6vHX3/u/J72N44c/nwVMsvSIuq0JZxwkvCc4BVwCh3rwEIX0eGbm3Np732qhTt6fQ94F+A5vB4GLDf3RtTjOGNcYfnD4T+nZ1nOp0G1AK/sMQht5+b2Slk6T5y99eBbwGvATUkvudryO591KIn9klb2+gJf0fiVQ90fk5d+T1sUy6HRqpjwxl3qZiZDQB+B3zW3Q+21zVFm3ehPS3M7Apgt7uvSW5uZwwZPZ+gkMQhgzvc/RzgCInDEm3J6DmFY/ALSBzSGAOcAlzWzhgyej4RZf0czOzLQCNwf0tTim5dnVOn55vLoVEFjE96PA6ojmksKZlZbxKBcb+7/z407zKzkvB8CbA7tLc1n/bax6VoT5e3AX9jZtuAJSQOUX0PKDazwhRjeGPc4fnBQB2dn2c6VQFV7r4qPF5KIkSydR+9B9jq7rXufhz4PXA+2b2PWvTEPmlrG2kTTtBfAXzYwzEkOj+nPXR+H7ctXcdP476R+C9xC4n/qlpOCs2Me1xJ4zPgHuB7rdpv58STbf8V7r+XE0/oPR/ah5I47j4k3LYCQ8Nzq0PflhN6l/fQ3C7gryfCf8uJJ+A+Fe7fwIkn4B4I92dy4km+LSRO8MWyP4FngNPD/a+E/ZOV+wg4D9gA9A/bWwx8Ohv3EW8+p5H2fdLWNtI4p/nARmBEq36d/v53dh+3O850/9LFeSNx5cQrJK4o+HLc42k1treTeBlYDrwUbpeTOJ74BLA5fG35QTbgx2Eu64DSpHX9HVAZbp9Iai8F1odlfkQHJ7i6cW4X8NfQOI3E1SiV4Qe3KLT3DY8rw/OnJS3/5TDmTSRdTRTH/gRmA2VhP/1P+AOTtfsI+Hfg5bDNe8MfnqzaR8CvSZyTOU7iP+Vre2KftLWNNM6pksT5hpa/Dz/t6ve/K/u4rZveES4iIpHl8jkNERHpZgoNERGJTKEhIiKRKTRERCQyhYaIiESm0BARkcgUGiIiEplCQ0REIvv/x5aO2Li/yxEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18b36c31b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Analysis for choosing optimum word freq cut off\n",
    "freqsL = list(range(1, 80, 5))\n",
    "num_words = []\n",
    "num_sents = []\n",
    "for x in freqsL:\n",
    "    filtered_words = dp.filter_words(word_freqs, x)\n",
    "    num_words.append(len(filtered_words))\n",
    "    num_sents.append(len(dp.get_sentences_with_words(clean_tweets, filtered_words)))\n",
    "plt.plot(num_words, num_sents)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocabulary size =  12614\n",
      "Number of sentences:  130283\n",
      "[['seriously', ',', 'why', 'is', 'it', 'so', 'difficult', 'to', 'find', 'a', 'decide', 'lamp', 'shade', 'around', 'here', '?', 'two', 'days', 'of', 'looking', 'and', 'still', 'no', 'winners'], ['is', 'looking', 'at', 'some', 'old', 'pictures', 'threedots', 'ah', '!', 'those', 'were', 'the', 'days', 'threedots'], ['on', 'a', 'new', 'work', 'out', 'routine', 'so', 'pooped', 'after', 'it', '!'], ['omg', 'my', 'head', 'feels', 'like', \"it's\", 'about', 'to', 'explode', 'threedots', 'i', 'miss', 'my', 'mummy'], ['my', 'temperature', 'is', 'going', 'back', 'up', 'i', 'just', 'want', 'to', 'get', 'better', '!'], ['so', 'sick', 'threedots', 'probably', \"shouldn't\", 'have', 'gone', 'to', 'work', '.', 'my', 'body', 'hates', 'me'], ['yeah', 'its', 'difficult', 'letting', 'you', 'down', 'threedots', 'knowing', 'its', 'all', 'my', 'fault', '.'], ['back', 'out', 'of', 'l', 'o', 'v', 'e', '.', 'ha', '!'], ['oh', 'no', '!', 'i', \"don't\", 'want', 'fever', 'now', 'too', '!'], ['theatre', 'was', 'much', 'better', 'than', 'expected', '.', 'quite', 'enjoyed', 'it', '.', 'then', 'again', ',', 'i', 'was', 'going', 'in', 'with', 'extremely', 'low', 'expectations']]\n"
     ]
    }
   ],
   "source": [
    "# let's consider tweets whose vocabulary matches with that of jokes and quotes dataset\n",
    "with open('./datasets/jokes_vocabulary.pickle', 'rb') as pickleFile:\n",
    "    jokes_vocabulary = pickle.load(pickleFile)\n",
    "    \n",
    "with open('./datasets/quotes_vocabulary.pickle', 'rb') as pickleFile:\n",
    "    quotes_vocabulary = pickle.load(pickleFile)\n",
    "\n",
    "final_vocabulary = jokes_vocabulary\n",
    "final_vocabulary.extend(quotes_vocabulary)\n",
    "final_vocabulary = list(set(final_vocabulary))\n",
    "print(\"vocabulary size = \", len(final_vocabulary))\n",
    "\n",
    "final_clean_tweets = dp.get_sentences_with_words(clean_tweets, final_vocabulary)\n",
    "print(\"Number of sentences: \", len(final_clean_tweets))\n",
    "print(final_clean_tweets[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final vocabulary size:  10805\n",
      "Number of sentences:  130250\n",
      "[\"sos certain times of the night bring in the most people and it's then that this happens quite frequently lately ! eos\", 'sos going to look in the refrigerator for a taste snack to eat eos', \"sos omg . i was just joking . now he deleted his twitter . great . i'm always the bad guy . eos\", 'sos lindsay lohan took a photo of herself topless with her blackberry then posted it to twitter for the world to see eos', 'sos praise the lord for very best friends ! just picked mine up from the airport . eos']\n",
      "['sos', 'certain', 'times', 'of', 'the', 'night', 'bring', 'in', 'most', 'people']\n"
     ]
    }
   ],
   "source": [
    "# Let's append sos and eos tokens to all sentences\n",
    "clean_tweets_final = [[\"sos\"] + tokens + [\"eos\"] for tokens in final_clean_tweets]\n",
    "clean_tweets_final = [' '.join(tokens) for tokens in clean_tweets_final]\n",
    "clean_tweets_final = list(set(clean_tweets_final))\n",
    "\n",
    "vocabulary_final = dp.filter_words(dp.get_vocabulary(dp.tokenize(clean_tweets_final)), 0)\n",
    "print(\"Final vocabulary size: \", len(vocabulary_final))\n",
    "print(\"Number of sentences: \", len(clean_tweets_final))\n",
    "print(clean_tweets_final[:5])\n",
    "print(vocabulary_final[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./datasets/tweets.pickle\",\"wb\") as fop:\n",
    "    pickle.dump(clean_tweets_final, fop)\n",
    "    \n",
    "with open(\"./datasets/tweets_vocabulary.pickle\",\"wb\") as fop:\n",
    "    pickle.dump(vocabulary_final, fop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"sos certain times of the night bring in the most people and it's then that this happens quite frequently lately ! eos\", 'sos going to look in the refrigerator for a taste snack to eat eos', \"sos omg . i was just joking . now he deleted his twitter . great . i'm always the bad guy . eos\", 'sos lindsay lohan took a photo of herself topless with her blackberry then posted it to twitter for the world to see eos', 'sos praise the lord for very best friends ! just picked mine up from the airport . eos', \"sos just totally baked cookies and figured out what i'm wearing to dinner . eos\", 'sos after facebook it seems hackers are now behind my twitter password . eos', \"sos i don't even know why my eyes are open right now . i was having a very strange dream and then i heard my poor sick nephew crying eos\", 'sos sleeping in my own bed tonight threedots at home ! eos', 'sos dropped my iphone it not working look for a replacement eos', 'sos the cat has settled himself on my lap and decided i am not going to work today . i can go with that eos', \"sos sleepy threedots driving home tonight i wish it wasn't over already eos\", 'sos haha just wiped out on some stairs . hurt my wrist a lil bit eos', 'sos so nice to recognize the spine . must sit up straight , you must ! eos', 'sos just had a delicious breakfast threedots it should be fathers day everyday threedots time for laundry now eos']\n"
     ]
    }
   ],
   "source": [
    "print(clean_tweets_final[:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}