data-preprocessing-quotes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "import matplotlib.pyplot as plt\n",
    "importlib.reload(dp)\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path1 = \"./data/author-quote.txt\"\n",
    "file_path2 = \"./data/quotes_all.csv\"\n",
    "file_path3 = \"./data/crowdtruth_proverbs.pickle\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36159\n",
      "['We have to do the impossible, but it is possible.', \"Don't limit a child to your own learning, for he was born in another time.\"]\n"
     ]
    }
   ],
   "source": [
    "quotes1 = list(set(dp.load_data(file_path1, header=False, sep='\\t')[1].values))\n",
    "print(len(quotes1))\n",
    "print(quotes1[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61071\n",
      "[\"Don't limit a child to your own learning, for he was born in another time.\", 'I am lucky to have advisers whom I trust.']\n"
     ]
    }
   ],
   "source": [
    "quotes2 = list(set(dp.load_data(file_path2, header=True, sep=';')[\"QUOTE\"]))\n",
    "print(len(quotes2))\n",
    "print(quotes2[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1019\n",
      "['Long Absent, Soon Forgotten', 'Never Do Things By Halves']\n"
     ]
    }
   ],
   "source": [
    "with open(file_path3, 'rb') as pickleFile:\n",
    "    quotes3 = pickle.load(pickleFile)\n",
    "    quotes3 = list(set(quotes3))\n",
    "print(len(quotes3))\n",
    "print(quotes3[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90019\n",
      "[\"Don't limit a child to your own learning, for he was born in another time.\", 'Racism, xenophobia and unfair discrimination have spawned slavery, when human beings have bought and sold and owned and branded fellow human beings as if they were so many beasts of burden.']\n"
     ]
    }
   ],
   "source": [
    "quotes = quotes1\n",
    "quotes.extend(quotes2)\n",
    "quotes.extend(quotes3)\n",
    "quotes = list(set(quotes))\n",
    "print(len(quotes))\n",
    "print(quotes[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88\n",
      "[' ', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '=', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '£', 'Â', 'â', '”', '€', '™']\n"
     ]
    }
   ],
   "source": [
    "unique_chars = dp.get_unique_chars(quotes)\n",
    "print(len(unique_chars))\n",
    "print(unique_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences containing each char is,\n",
      "{' ': 90019, '!': 1452, '\"': 8, '#': 5, '$': 148, '%': 101, '&': 62, \"'\": 37355, '(': 133, ')': 139, '*': 4, '+': 5, ',': 54194, '-': 11056, '.': 87913, '/': 126, '0': 2245, '1': 2197, '2': 1303, '3': 792, '4': 678, '5': 941, '6': 618, '7': 499, '8': 586, '9': 896, ':': 2481, ';': 2040, '=': 7, '?': 2919, 'A': 15365, 'B': 8410, 'C': 5887, 'D': 4033, 'E': 4650, 'F': 4530, 'G': 4981, 'H': 5544, 'I': 47185, 'J': 2109, 'K': 1159, 'L': 4417, 'M': 8826, 'N': 5000, 'O': 4642, 'P': 5103, 'Q': 212, 'R': 2867, 'S': 9180, 'T': 23280, 'U': 1728, 'V': 1215, 'W': 13138, 'X': 78, 'Y': 5268, 'Z': 137, 'a': 88790, 'b': 64007, 'c': 74702, 'd': 81209, 'e': 89750, 'f': 72721, 'g': 72782, 'h': 85573, 'i': 88186, 'j': 11766, 'k': 47145, 'l': 83349, 'm': 77446, 'n': 88486, 'o': 89115, 'p': 65696, 'q': 5608, 'r': 87411, 's': 88140, 't': 89013, 'u': 79087, 'v': 59699, 'w': 70813, 'x': 11654, 'y': 75113, 'z': 5700, '£': 4, 'Â': 4, 'â': 1, '”': 1, '€': 1, '™': 1}\n"
     ]
    }
   ],
   "source": [
    "dp.get_char_presence(quotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of quotes =  90019\n",
      "83449\n",
      "[\"Don't limit a child to your own learning, for he was born in another time.\", 'Racism, xenophobia and unfair discrimination have spawned slavery, when human beings have bought and sold and owned and branded fellow human beings as if they were so many beasts of burden.', 'I am lucky to have advisers whom I trust.', \"It took me so many years to move out. I'm definitely a bit of a Peter Pan, reluctant to grow up. It all seemed really nice at home-why change it? Part of me would prefer not to have any responsibility whatsoever.\", \"There was a sense that there was a lot of word of mouth happening with 'The Memory Keeper's Daughter,' even in hardcover.\", \"My father opened a restaurant. It's so amazing... it's so freaking delicious, but I'm telling you I gain five pounds every time I go in there.\", 'Men take only their needs into consideration - never their abilities.', \"He that does good for good's sake seeks neither paradise nor reward, but he is sure of both in the end.\", 'To walk around with an ego is a bad thing. To have confidence in yourself is a great thing.', \"It's so bizarre, I'm not scared of snakes or spiders. But I'm scared of butterflies. There is something eerie about them. Something weird!\"]\n"
     ]
    }
   ],
   "source": [
    "print(\"Total number of quotes = \", len(quotes))\n",
    "filter_quotes = dp.filter_data(quotes, \"^[a-zA-Z !\\\"',.?\\-\\:\\*]+$\")\n",
    "print(len(filter_quotes))\n",
    "print(filter_quotes[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n",
      "[' ', '!', '\"', \"'\", '*', ',', '-', '.', ':', '?', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
      "83449\n",
      "[[\"don't\", 'limit', 'a', 'child', 'to', 'your', 'own', 'learning', ',', 'for', 'he', 'was', 'born', 'in', 'another', 'time', '.'], ['racism', ',', 'xenophobia', 'and', 'unfair', 'discrimination', 'have', 'spawned', 'slavery', ',', 'when', 'human', 'beings', 'have', 'bought', 'and', 'sold', 'and', 'owned', 'and', 'branded', 'fellow', 'human', 'beings', 'as', 'if', 'they', 'were', 'so', 'many', 'beasts', 'of', 'burden', '.']]\n",
      "min of length:  2\n",
      "max of length:  108\n",
      "average length:  27.517166173351388\n",
      "mode of length:  14\n"
     ]
    }
   ],
   "source": [
    "pad_chars = [' ', '!', '\"', '*', ',', '-', ':', '?']\n",
    "clean_quotes = dp.clean_data(quotes, \"^[a-zA-Z !\\\"'*,\\-.:?]+$\", pad_chars)\n",
    "\n",
    "unique_chars = dp.get_unique_chars(clean_quotes)\n",
    "print(len(unique_chars))\n",
    "print(unique_chars)\n",
    "\n",
    "clean_quotes = dp.tokenize(clean_quotes)\n",
    "\n",
    "print(len(clean_quotes))\n",
    "print(clean_quotes[:2])\n",
    "\n",
    "# stats of number of tokens in the sentence\n",
    "dp.get_len_stats(clean_quotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80036\n"
     ]
    }
   ],
   "source": [
    "MIN_NUM_TOKENS = 6\n",
    "MAX_NUM_TOKENS = 60\n",
    "clean_quotes = dp.filter_data_on_length(clean_quotes, MIN_NUM_TOKENS, MAX_NUM_TOKENS)\n",
    "print(len(clean_quotes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freqs = dp.get_vocabulary(clean_quotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total vocabulary:  34977\n",
      "average of frequency:  60.49644051805472\n",
      "mode of frequency:  1\n"
     ]
    }
   ],
   "source": [
    "dp.get_vocabulary_stats(word_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of filtered words =  5299\n",
      "29198\n",
      "[[\"don't\", 'limit', 'a', 'child', 'to', 'your', 'own', 'learning', ',', 'for', 'he', 'was', 'born', 'in', 'another', 'time', '.'], ['men', 'take', 'only', 'their', 'needs', 'into', 'consideration', '-', 'never', 'their', 'abilities', '.'], ['to', 'walk', 'around', 'with', 'an', 'ego', 'is', 'a', 'bad', 'thing', '.', 'to', 'have', 'confidence', 'in', 'yourself', 'is', 'a', 'great', 'thing', '.'], ['an', 'ill', 'wound', 'is', 'cured', ',', 'not', 'an', 'ill', 'name'], [\"i'm\", 'so', 'happy', 'to', 'be', 'able', 'to', 'give', 'kids', 'the', 'opportunity', 'to', 'learn', 'about', 'amazing', 'world', 'of', 'dance', 'and', 'music', 'that', \"i've\", 'have', 'been', 'lucky', 'enough', 'to', 'make', 'such', 'a', 'big', 'part', 'of', 'my', 'own', 'life', '.']]\n"
     ]
    }
   ],
   "source": [
    "filtered_words = dp.filter_words(word_freqs, 20)\n",
    "print(\"Number of filtered words = \", len(filtered_words))\n",
    "sents = dp.get_sentences_with_words(clean_quotes, filtered_words)\n",
    "print(len(sents))\n",
    "print(sents[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8VOW1//HPghDu9ztJEFCuoggMQtVaqxYBrWhVsLaVWi091lbb09Of2tpjr+dl23Nq6znWHqseobUlEbViK0WK0qtiJiBylwDKhAAJAgGEhFzW7495qJEEEkKSmUm+79drXrNn7WfPrNm5rNn7eebZ5u6IiIhU1ybRCYiISPJRcRARkRpUHEREpAYVBxERqUHFQUREalBxEBGRGlQcRESkBhUHERGpQcVBRERqSEt0Ag3Vp08fHzJkSKLTEBFJGXl5eXvcvW992qZscRgyZAjRaDTRaYiIpAwze6e+bXVaSUREalBxEBGRGlQcRESkBhUHERGpQcVBRERqqFdxMLOvmtk6M1trZr81sw5mNtTMVpjZZjPLNrP00LZ9eJwf1g+p9jz3hvgmM7uiWnxaiOWb2T2N/SZFROTU1FkczCwDuBOIuPtYoC1wI/BD4EF3Hw7sA24Nm9wK7HP3s4AHQzvMbEzY7mxgGvBzM2trZm2Bh4HpwBjgk6GtiIgkSH1PK6UBHc0sDegE7AQuBRaG9fOAa8LyzPCYsP4yM7MQX+DuZe6+DcgHzg+3fHff6u5HgQWhrYiIBOWVVby0bhePLN/SLK9X55fg3H2Hmf0nsB04ArwE5AH73b0iNCsAMsJyBhAL21aYWQnQO8Rfq/bU1beJHRefXFsuZjYXmAswePDgulIXEUl5W4sPkR2N8UzeDvYcKiOjR0c+d9EQ2qe1bdLXrbM4mFlP4p/khwL7gaeJnwI6nh/b5ATrThSv7ejFa4nh7o8CjwJEIpFa24iIpLrDRyt4cc0usnO3k/v2Ptq2MT46sh83TsrikpF9SWvb9GOJ6jN9xuXANncvBjCzZ4ELgB5mlhaOHjKBwtC+AMgCCsJpqO7A3mrxY6pvc6K4iEir4O6sLighOzfGC6sLOVRWwdA+nbl72iium5BBv24dmjWf+hSH7cAUM+tE/LTSZUAUeAW4nngfwRzg+dB+UXj8alj/sru7mS0CfmNmPwEGAcOB14kfUQw3s6HADuKd1jc1ztsTEUlue987ynOrdpCTG2PT7oN0aNeGK88ZxOxJWUwa0pN4l23zq0+fwwozWwisBCqAVcRP7fwBWGBm3w+xx8MmjwO/MrN84kcMN4bnWWdmOcD68Dx3uHslgJl9CVhCfCTUE+6+rvHeoohIcqmqcv6Wv4fsaIyl63ZztLKKcZnd+cG1Y/n4uEF069Au0Sli7ql56j4SibhmZRWRVFKw7zBPRwtYmFfAjv1H6NGpHdeOz2D2pCxGDejW5K9vZnnuHqlP25SdsltEJBWUVVSydP1usnNj/C1/DwAXndWHe2eM4mNj+jf5qKOGUnEQEWkCG3cdIDs3xu9W7WDf4XIyenTkzkuHc0Mkk8yenRKdXp1UHEREGsnB0nJeWL2T7NztrC4ooV1bY+qYAcyelMWFZ/WhbZvEdC43hIqDiMhpcHdy395Hdm6MF9fs5Eh5JSP7d+VbV43h2vEZ9OqcnugUG0TFQUSkAYoOlvJM3g6ejsbYuuc9urRP45rQuTwus3vChqA2FhUHEZF6qqisYvmmYrKjMV7eWERllTNpSE9uv+RMrjx3IJ3SW86/1JbzTkREmsi2Pe+RE43xTF4BRQfL6NOlPbd9eCizIlmc2bdLotNrEioOIiK1OHK0ksVrd5KdG2PFtr20MfjoyH7MmpTFpaP60a4Z5jdKJBUHEZHA3VmzIz6/0aI3CjlYVsEZvTvx9StGcv3ETPo38/xGiaTiICKt3v7DR/ndqh0syI2xcddB2qe1YcY5A5k9KYvJQ3ulfOdyQ6g4iEirVFXl/GPLu2RHYyxZt4ujFVWck9Gd710zlqvHDaJ7x8TPb5RIKg4i0qoU7j/CwrwCcqIxCvYdoXvHdtx0/mBmRbIYM6jp5zdKFSoOItLiHa2o4k8b4vMb/WVzMe5w4Vm9+foVI7ni7AF0aJec8xslkoqDiLRYb+0+SHZujOdW7WDve0cZ2L0DX/7oWdwQySKrV/LPb5RIKg4i0qIcKqvg96sLyY7GWLV9P+3aGpeP7s+sSVlcPLxvSs1vlEgqDiKS8tydvHfi8xv9Yc1ODh+tZHi/Ltx35WiuHZ9B7y7tE51iylFxEJGUtedQGc+uLCA7N8aW4vfonN6Wj587iNnnZzE+q0erHILaWFQcRCSlVFRW8ZfNxWTnxli2oYiKKmfiGT350XXx+Y06t9e/tcagvSgiKWH7u4fJicZYmFfArgOl9O6czucuGsqsSCZn9eua6PRanDqLg5mNBLKrhYYB/w7MD/EhwNvALHffZ/HjuJ8BM4DDwGfdfWV4rjnAfeF5vu/u80J8IvAk0BF4EbjLU/Xi1iLSaErLK/nj2l1k58Z4deu7tDH4yIi+fPvqMVw6qj/paS17fqNEqrM4uPsm4DwAM2sL7ACeA+4Blrn7A2Z2T3h8NzAdGB5uk4FHgMlm1gu4H4gADuSZ2SJ33xfazAVeI14cpgGLG/F9ikgKWbujhJxo/BKbB0oryOrVkX+bOoLrJmYysHvHRKfXKpzqaaXLgC3u/o6ZzQQuCfF5wHLixWEmMD988n/NzHqY2cDQdqm77wUws6XANDNbDnRz91dDfD5wDSoOIq1KyeFynl+9g+zcGOsKD5Ce1obpYwcwO5LFlGG9aaMhqM3qVIvDjcBvw3J/d98J4O47zaxfiGcAsWrbFITYyeIFtcRFpIWrqnJe2xqf3+iPa3dRVlHF2YO68d2ZZzNzXAbdO7Xu+Y0Sqd7FwczSgauBe+tqWkvMGxCvLYe5xE8/MXjw4DrSEJFktauklIV5MXKiBWzfe5iuHdKYFcli9qQsxmZ0T3R6wqkdOUwHVrr77vB4t5kNDEcNA4GiEC8AsqptlwkUhvglx8WXh3hmLe1rcPdHgUcBIpGIOqxFUkh5ZRXLNhSRnbudP79VTJXDh4b15l8/NoJpYzW/UbI5leLwSd4/pQSwCJgDPBDun68W/5KZLSDeIV0SCsgS4D/MrGdoNxW41933mtlBM5sCrABuBv67we9IRJJKftEhcqIxnl1ZwJ5DR+nfrT23X3ImsyJZnNG7c6LTkxOoV3Ews07Ax4AvVAs/AOSY2a3AduCGEH+R+DDWfOJDWW8BCEXge0BuaPfdY53TwO28P5R1MeqMFklp75VV8Ic3d5IdjZH3zj7S2hiXje7H7DC/UVoLv8RmS2Cp+nWCSCTi0Wg00WmISODurIrtJyc3xgurC3nvaCXD+nZmdiSLT0zIpG9XzW+UaGaW5+6R+rTVN6RF5LS8e6iM51bFh6BuLjpEx3Ztuerc+CU2J57RU/MbpSgVBxE5ZZVVzl/D/EZ/2rCb8kpn/OAePPCJc7hq3CC6aH6jlKefoIjUW2zvYZ4O8xsVlpTSq3M6N39oCLMnZTGiv+Y3aklUHETkpErLK3lp/W6yc7fz9/x3MYOLh/flvqvGcPlozW/UUqk4iEit1hceICcav8RmyZFyMnp05KuXj+D6SCYZPTS/UUun4iAi/1RypJxFqwvJyY2xZkcJ6W3bcEWY3+iCMzW/UWui4iDSyrk7K7btJSdcYrOsoopRA7py/8fHcM15GfTsnJ7oFCUBVBxEWqndB0pZmFfA09EYb797mK7t07h+YiazJ2VxTkZ3DUFt5VQcRFqR8soqXtlYRE40xiubiqmsciYP7cWdlw1n+tiBdEzX/EYSp+Ig0gpsLT5EdjTGM3k72HOojL5d2zP34mHMimQxtI/mN5KaVBxEWqjDRyt4cc0usnO3k/v2Ptq2MT46sh83TsrikpGa30hOTsVBpAVxd1YXlJAd5jc6VFbB0D6duXvaKK6bkEG/bh0SnaKkCBUHkRZg33tH/zm/0abdB+nQrg1XnjOI2ZOymDRE8xvJqVNxEElRVVXO3/L3kB2NsXTdbo5WVjEuszs/uHYsHx83iG4ddIlNaTgVB5EUU7DvME9HC1iYV8CO/Ufo0akdn5oymNmTshg1oFui05MWQsVBJAWUVVSydP1usnNj/C1/DwAXndWHe2eM4mNj+tM+TUNQpXGpOIgksY27DpCdG+N3q3aw73B8fqM7Lx3ODZFMMnt2SnR60oKpOIgkmYOl5byweifZudtZXVBCu7bG1DEDmD0piwvP6kNbzW8kzUDFQSQJuDu5b+8jOzfGi2t2cqS8kpH9u/Ktq8Zw7fgMeml+I2lmKg4iCVR0sJRnV+4gJzfG1j3v0aV9GteMz2D2pCzGZWp+I0mcehUHM+sBPAaMBRz4HLAJyAaGAG8Ds9x9n8V/m38GzAAOA59195XheeYA94Wn/b67zwvxicCTQEfgReAud/fTf3siyaeisorlm4rJjsZ4eWMRlVXOpCE9uf2SM7ny3IF0StdnNkm8+v4W/gz4o7tfb2bpQCfgG8Ayd3/AzO4B7gHuBqYDw8NtMvAIMNnMegH3AxHiBSbPzBa5+77QZi7wGvHiMA1Y3EjvUSQpbNvzHjnRGM/kFVB0sIw+Xdpz24eHMiuSxZl9uyQ6PZEPqLM4mFk34GLgswDufhQ4amYzgUtCs3nAcuLFYSYwP3zyf83MepjZwNB2qbvvDc+7FJhmZsuBbu7+aojPB65BxUFagCNHK1m8difZuTFWbNtLG4OPjuzHrElZXDqqH+00v5EkqfocOQwDioH/M7NxQB5wF9Df3XcCuPtOM+sX2mcAsWrbF4TYyeIFtcRFUpK7s2ZHfH6jRW8UcrCsgjN6d+LrV4zk+omZ9Nf8RpIC6lMc0oAJwJfdfYWZ/Yz4KaQTqa0HzRsQr/nEZnOJn35i8ODBJ8tZpNntP3yU363aQXa0gA07D9A+rQ0zzhnI7ElZTB7aS53LklLqUxwKgAJ3XxEeLyReHHab2cBw1DAQKKrWPqva9plAYYhfclx8eYhn1tK+Bnd/FHgUIBKJqMNaEq6qyvnHlnfJjsZYsm4XRyuqOCejO9+7ZixXjxtE946a30hSU53Fwd13mVnMzEa6+ybgMmB9uM0BHgj3z4dNFgFfMrMFxDukS0IBWQL8h5n1DO2mAve6+14zO2hmU4AVwM3AfzfiexRpdIX7j7Awr4CcaIyCfUfo3rEdN50/mFmRLMYM0vxGkvrqO1rpy8BTYaTSVuAWoA2QY2a3AtuBG0LbF4kPY80nPpT1FoBQBL4H5IZ23z3WOQ3czvtDWRejzmhJQkcrqvjThvj8Rn/ZXIw7XHhWb75+xUiuOHsAHdppfiNpOSxVv04QiUQ8Go0mOg1pBd7afZDs3BjPrdrB3veOMrB7B26YmMkNkSyyeml+I0kdZpbn7pH6tNW3bURqcaisgt+vLiQ7GmPV9v20a2tcPro/syZlcfHwvprfSFo8FQeRwN1ZuT0+v9Hv39zJ4aOVDO/XhfuuHM214zPo3aV9olMUaTYqDtLq7TlUxrMrC8jOjbGl+D06p7fl4+cOYvb5WYzP6qEhqNIqqThIq1RRWcVfNheTnRtj2YYiKqqciWf05EfXxec36txefxrSuukvQFqV7e8eJicaY2FeAbsOlNK7czqfu2gosyKZnNWva6LTE0kaKg7S4pWWV/LHtbvIzo3x6tZ3aWPwkRF9+fbVY7h0VH/S0zS/kcjxVBykxVq7o4ScaPwSmwdKK8jq1ZF/mzqC6yZmMrB7x0SnJ5LUVBykRSk5XM7zq3eQnRtjXeEB0tPaMH3sAGZHspgyrDdtNARVpF5UHCTlVVU5r217l5zcGIvX7qKsooqzB3XjuzPPZua4DLp30vxGIqdKxUFS1q6SUhbmxciJFrB972G6dkhjViSL2ZOyGJvRPdHpiaQ0FQdJKeWVVSzbUER27nb+/FYxVQ4fGtabf/3YCKaN1fxGIo1FxUFSQn7RIXKiMZ5dWcCeQ0fp3609t19yJrMiWZzRu3Oi0xNpcVQcJGm9V1bBH9bEL7GZ984+0toYl43ux+wwv1GaLrEp0mRUHCSpuDurYvvJyY3xwupC3jtaybC+nbl3+ig+MSGTvl01v5FIc1BxkKTw7qEynlsVH4K6uegQHdu15apz45fYnHhGT81vJNLMVBwkYSqrnL9uLiYnGmPp+t2UVzrjB/fggU+cw1XjBtFF8xuJJIz++qTZxfYe5ukwv1FhSSm9Oqdz84eGMHtSFiP6a34jkWSg4iDNorS8kpfW7yYnN8bf8vdgBhcP78t9V43h8tGa30gk2ag4SJNaX3iAnGj8EpslR8rJ6NGRr14+gusjmWT00PxGIslKxUEa3YHScha9UUhONMabBSWkt23DFWF+owvO1PxGIqmgXsXBzN4GDgKVQIW7R8ysF5ANDAHeBma5+z6LDyv5GTADOAx81t1XhueZA9wXnvb77j4vxCcCTwIdgReBu9zdG+H9STPaWnyI/3k5nxfX7qS0vIpRA7py/8fHcM15GfTsnJ7o9ETkFJzKkcNH3X1Ptcf3AMvc/QEzuyc8vhuYDgwPt8nAI8DkUEzuByKAA3lmtsjd94U2c4HXiBeHacDi03pn0qzW7ijhM4+voLzSuW5CJrMnZXFORncNQRVJUadzWmkmcElYngcsJ14cZgLzwyf/18ysh5kNDG2XuvteADNbCkwzs+VAN3d/NcTnA9eg4pAy8t7Zy2efyKVbx3Y8ddtkhvTRdBYiqa6+Q0QceMnM8sxsboj1d/edAOG+X4hnALFq2xaE2MniBbXEazCzuWYWNbNocXFxPVOXpvS3zXv49GOv07dre57+lw+pMIi0EPU9crjQ3QvNrB+w1Mw2nqRtbecRvAHxmkH3R4FHASKRiPokEuxP63fzxadWMqxvZ35162RNbSHSgtTryMHdC8N9EfAccD6wO5wuItwXheYFQFa1zTOBwjrimbXEJYm9sLqQf/l1HqMHdmXB3CkqDCItTJ3Fwcw6m1nXY8vAVGAtsAiYE5rNAZ4Py4uAmy1uClASTjstAaaaWU8z6xmeZ0lYd9DMpoSRTjdXey5JQjm5Me5csIoJZ/Tk17dNpkcnjUQSaWnqc1qpP/BcGHWSBvzG3f9oZrlAjpndCmwHbgjtXyQ+jDWf+FDWWwDcfa+ZfQ/IDe2+e6xzGrid94eyLkad0Unr//6+je+8sJ6LR/Tlfz89kY7puriOSEtkqfp1gkgk4tFoNNFptCoPv5LPj5ds4oqz+/PQJ8fTPk2FQSSVmFmeu0fq01bfkJY6uTs/WrKJR5Zv4drxGfz4+nN1oR2RFk7FQU6qqsr5zgvrmPfqO9w0eTDfnzlW01+ItAIqDnJClVXO3c+8ycK8Aj7/4aF8Y8ZofeNZpJVQcZBaHa2o4qvZb/CHNTv56uUjuPOys1QYRFoRFQepobS8ki8+tZKXNxbxzRmj+fzFwxKdkog0MxUH+YD3yiq4bV6U17a9yw+uHcunJp+R6JREJAFUHOSfSo6Uc8v/vc7qghJ+Mmsc147PrHsjEWmRVBwEgHcPlfGZx19nc9FBHr5pAtPGDkh0SiKSQCoOwq6SUj712Gvs2H+Ex+ZM4iMj+iY6JRFJMBWHVi629zA3PfYa+94rZ94t5zN5WO9EpyQiSUDFoRXLLzrEpx9bwZHySp66bTLjsnokOiURSRIqDq3U+sIDfObxFZgZ2V+YwqgB3RKdkogkEU2Q0wqt3L6PGx99lfZpbchRYRCRWujIoZV5dcu73Dovl75d2/PUbZPJ7Nkp0SmJSBJScWhFXtlYxL/8Oo/BvTrx1G2T6detQ6JTEpEkpeLQSixes5M7F6xi5ICuzP/cZHp11tXbROTE1OfQCizMK+CO36xkXGYPfvP5KSoMIlInHTm0cL969W2+9fw6LjqrD4/ePJFO6fqRi0jd9J+iBfvFn7fwwOKNXD66P/9z03g6tNNlPUWkfup9WsnM2prZKjP7fXg81MxWmNlmM8s2s/QQbx8e54f1Q6o9x70hvsnMrqgWnxZi+WZ2T+O9vdbJ3fnJS5t4YPFGPj5uEI98eoIKg4icklPpc7gL2FDt8Q+BB919OLAPuDXEbwX2uftZwIOhHWY2BrgROBuYBvw8FJy2wMPAdGAM8MnQVhrA3fne7zfw0Mv5zI5k8dPZ59FO13sWkVNUr/8aZpYJXAk8Fh4bcCmwMDSZB1wTlmeGx4T1l4X2M4EF7l7m7tuAfOD8cMt3963ufhRYENrKKaqscu59dg1P/H0bt1w4hAeuO4e2ut6ziDRAfT9S/hT4f0BVeNwb2O/uFeFxAZARljOAGEBYXxLa/zN+3DYnisspKK+MX9ZzQW6ML196Fv9+1Rhd1lNEGqzO4mBmVwFF7p5XPVxLU69j3anGa8tlrplFzSxaXFx8kqxbl8oq50u/Wcmi1YXcPW0UX5s6UoVBRE5LfUYrXQhcbWYzgA5AN+JHEj3MLC0cHWQChaF9AZAFFJhZGtAd2Fstfkz1bU4U/wB3fxR4FCASidRaQFqjBxZvYMm63XzrqjHcetHQRKcjIi1AnUcO7n6vu2e6+xDiHcovu/ungFeA60OzOcDzYXlReExY/7K7e4jfGEYzDQWGA68DucDwMPopPbzGokZ5d61ATjTGL/+6jc9eMESFQUQazel8z+FuYIGZfR9YBTwe4o8DvzKzfOJHDDcCuPs6M8sB1gMVwB3uXglgZl8ClgBtgSfcfd1p5NVq5L69l28+t4aLzurDfVeOTnQ6ItKCWPxDfeqJRCIejUYTnUbCxPYeZubDf6dHx3Y898UL6d6pXaJTEpEkZ2Z57h6pT1sNgE9Bh8oq+Pz8KBWVVTw2J6LCICKNTtNnpJiqKucrC95gc9EhnrxlEsP6dkl0SiLSAunIIcX8+KVN/GnDbr515Wg+PLxvotMRkRZKxSGFPLeqgEeWb+GmyYOZc8GQRKcjIi2YikOKWLl9H3c/s4Ypw3rxnavP1pfcRKRJqTikgML9R5g7P48B3TrwyKcmaiI9EWly6pBOcoePxkcmlZZX8tvPT6anruImIs1AH0GTWFWV87Wc1WzYeYD//uR4hvfvmuiURKSVUHFIYj9dtpnFa3fxjRmj+eiofolOR0RaERWHJPXC6kIeWraZGyZmas4kEWl2Kg5J6M2C/fzb06uZNKQn3792rEYmiUizU3FIMrsPlPL5+VH6dGnPI5+eSPs0XftZRJqfRislkdLySubOj3KwtIJnbr+APl3aJzolEWmlVByShLvz9YVv8uaOEv730xMZPbBbolMSkVZMp5WSxM+Xb+GF1YV8/YqRTD17QKLTEZFWTsUhCby+bS//9dImrh43iNs/cmai0xERUXFItJLD5XxlwSoG9+rEf3ziHI1MEpGkoD6HBHJ37n3uTYoOlvHM7RfQpb1+HCKSHHTkkEA50RgvrtnF16aOZFxWj0SnIyLyTyoOCbKl+BDfXrSeC87szRcuHpbodEREPqDO4mBmHczsdTNbbWbrzOw7IT7UzFaY2WYzyzaz9BBvHx7nh/VDqj3XvSG+ycyuqBafFmL5ZnZP47/N5FJWUcmdv11Fh3Zt+Mms82jTRv0MIpJc6nPkUAZc6u7jgPOAaWY2Bfgh8KC7Dwf2AbeG9rcC+9z9LODB0A4zGwPcCJwNTAN+bmZtzawt8DAwHRgDfDK0bbH+c8km1hUe4IfXncuA7h0SnY6ISA11FgePOxQetgs3By4FFob4POCasDwzPCasv8ziQ3BmAgvcvczdtwH5wPnhlu/uW939KLAgtG2R/vJWMb/86zY+PWWwvs8gIkmrXn0O4RP+G0ARsBTYAux394rQpADICMsZQAwgrC8BelePH7fNieItzp5DZfxrzmqG9+vCfVe26IMjEUlx9SoO7l7p7ucBmcQ/6Y+urVm4r+0EujcgXoOZzTWzqJlFi4uL6048ibg7/2/hmxwoLeehT46nQztNqCciyeuURiu5+35gOTAF6GFmxwbmZwKFYbkAyAII67sDe6vHj9vmRPHaXv9Rd4+4e6Rv376nknrCzfvH27y8sYhvTB+leZNEJOnVZ7RSXzPrEZY7ApcDG4BXgOtDsznA82F5UXhMWP+yu3uI3xhGMw0FhgOvA7nA8DD6KZ14p/WixnhzyWLDzgP8x+KNXDqqH3MuGJLodERE6lSfr+QOBOaFUUVtgBx3/72ZrQcWmNn3gVXA46H948CvzCyf+BHDjQDuvs7McoD1QAVwh7tXApjZl4AlQFvgCXdf12jvMMFKy+PDVrt3bMePrz9X02OISEqw+If61BOJRDwajSY6jTr94A/r+eVftzH/c+dz8YjUOhUmIi2LmeW5e6Q+bfUN6SaU984+HvvbNj41ebAKg4ikFBWHJlJaXsnXF65mUPeO3DujtsFdIiLJS9OANpEH//QWW4vf41e3nq/ZVkUk5ejIoQm8EdvPL/+ylRsnZfHh4TqdJCKpR8WhkZVVVPL1p1fTv1sHvnGlTieJSGrS+Y5G9tCyzWwuOsSTt0yiW4d2iU5HRKRBdOTQiN4s2M8v/ryVGyZmcsnIfolOR0SkwVQcGkn8dNKb9OmSzn1XaVI9EUltOq3USB5+OZ9Nuw/yxGcjdO+o00kiktp05NAINu46wM+Xb+ETEzK4dFT/RKcjInLaVBxOU2WVc88za+jesR3f0jUaRKSFUHE4TU+teIc3Yvv51lVj6Nk5PdHpiIg0ChWH07CrpJQf/XETHx7eh5nnDUp0OiIijUbF4TTcv2gtFVVV/OCaczQVt4i0KCoODfTSul0sWbebuy4bweDenRKdjohIo1JxaICDpeX8+/PrGDWgK7d9eGii0xERaXT6nkMD/NdLb7H7YCmPfHoC7dqqvopIy6P/bKdo7Y4S5r/6Np+ZcgbjB/dMdDoiIk1CxeEUVFU533p+Lb06p/O1qSMTnY6ISJNRcTgFC1cWsGr7fu6dPlpTZIhIi1ZncTCzLDN7xcw2mNk6M7srxHuZ2VIz2xzue4a4mdlDZpZvZm+a2YRqzzUntN9sZnOqxSea2ZqwzUOWhONCSw6X88DijUTO6MknJmQL2A4MAAAKP0lEQVQkOh0RkSZVnyOHCuBr7j4amALcYWZjgHuAZe4+HFgWHgNMB4aH21zgEYgXE+B+YDJwPnD/sYIS2syttt20039rjes/X9rE/sNH+e7MsfpOg4i0eHUWB3ff6e4rw/JBYAOQAcwE5oVm84BrwvJMYL7HvQb0MLOBwBXAUnff6+77gKXAtLCum7u/6u4OzK/2XElh7Y4Sfr3iHW7+0BDGDOqW6HRERJrcKfU5mNkQYDywAujv7jshXkCAY1e3yQBi1TYrCLGTxQtqidf2+nPNLGpm0eLi4lNJvcGOdUL37pzOVz82olleU0Qk0epdHMysC/AM8BV3P3CyprXEvAHxmkH3R9094u6Rvn371pVyo3gmdELfo05oEWlF6lUczKwd8cLwlLs/G8K7wykhwn1RiBcAWdU2zwQK64hn1hJPuJIj8U7oiWf05BPj1QktIq1HfUYrGfA4sMHdf1Jt1SLg2IijOcDz1eI3h1FLU4CScNppCTDVzHqGjuipwJKw7qCZTQmvdXO150qoB5e+xd7DR/nO1WfTpo06oUWk9ajP9BkXAp8B1pjZGyH2DeABIMfMbgW2AzeEdS8CM4B84DBwC4C77zWz7wG5od133X1vWL4deBLoCCwOt4TauOsA8199m09NHszYjO6JTkdEpFnVWRzc/W/U3i8AcFkt7R244wTP9QTwRC3xKDC2rlya04//uIku7dP42sf0TWgRaX30Dela5L2zj2Ubi/jCR87U1d1EpFVScTiOu/PjJRvp0yWdWy4ckuh0REQSQsXhOH/Pf5fXtu7ljo+eRad0zWguIq2TikM17s6PX9rEoO4duGny4ESnIyKSMCoO1SxZt5vVsf185fIRtE9rm+h0REQSRsUhqKxy/uulTZzZt7NmXRWRVk/FIfjdqh1sLjrE16aOJE2X/hSRVk7/BYGjFVX8dNlbjM3oxvSxAxKdjohIwqk4ADnRGLG9R/ja1JG6VoOICCoOlJZX8j8v5xM5oyeXjGiemV5FRJJdqy8OC17fzq4Dpfzrx0boqEFEJGjVxaG0vJKfL9/C+UN78aEzeyc6HRGRpNGqi8OC17dTdLCMr1w+XEcNIiLVtNriUFZRyS/+vJXzh/TigjP7JDodEZGk0mqLw9PRAnYdKOWuy4cnOhURkaTTKovD0YoqHlm+hQmDe3CB+hpERGpolcXh2ZUF7Nh/hDsvU1+DiEhtWl1xqKis4uHl+Zyb2Z2P6HsNIiK1anXFYdHqQmJ7j/DlS3XUICJyInUWBzN7wsyKzGxttVgvM1tqZpvDfc8QNzN7yMzyzexNM5tQbZs5of1mM5tTLT7RzNaEbR6yJvyPXVXl/Hz5FkYN6Mplo/o11cuIiKS8+hw5PAlMOy52D7DM3YcDy8JjgOnA8HCbCzwC8WIC3A9MBs4H7j9WUEKbudW2O/61Gs3h8komDu7Jly8dTps2OmoQETmROouDu/8F2HtceCYwLyzPA66pFp/vca8BPcxsIHAFsNTd97r7PmApMC2s6+bur7q7A/OrPVej69I+jR9efy5XnjuwqV5CRKRFaGifQ3933wkQ7o+do8kAYtXaFYTYyeIFtcRFRCSBGrtDurZzNd6AeO1PbjbXzKJmFi0uLm5giiIiUpeGFofd4ZQQ4b4oxAuArGrtMoHCOuKZtcRr5e6PunvE3SN9+2oYqohIU2locVgEHBtxNAd4vlr85jBqaQpQEk47LQGmmlnP0BE9FVgS1h00sylhlNLN1Z5LREQSJK2uBmb2W+ASoI+ZFRAfdfQAkGNmtwLbgRtC8xeBGUA+cBi4BcDd95rZ94Dc0O677n6sk/t24iOiOgKLw01ERBLI4oOEUk8kEvFoNJroNEREUoaZ5bl7pD5tW903pEVEpG4qDiIiUkPKnlYys2LgnUTnUQ99gD2JTqKBUjV35d28lHfzOp28z3D3eg31TNnikCrMLFrfc3zJJlVzV97NS3k3r+bKW6eVRESkBhUHERGpQcWh6T2a6AROQ6rmrrybl/JuXs2St/ocRESkBh05iIhIDSoODWRmb4cr2L1hZtEQa7Qr5DVinil5Jb8T5P1tM9sR9vkbZjaj2rp7Qw6bzOyKavFpIZZvZvdUiw81sxXh/WSbWXoj5Z1lZq+Y2QYzW2dmd4V4Uu/zk+Sd1PvczDqY2etmtjrk/Z2TvZaZtQ+P88P6IQ19P02U95Nmtq3a/j4vxJv/98TddWvADXgb6HNc7EfAPWH5HuCHYXkG8TmjDJgCrAjxXsDWcN8zLPds5DwvBiYAa5siT+B14ENhm8XA9CbM+9vAv9XSdgywGmgPDAW2AG3DbQswDEgPbcaEbXKAG8PyL4DbGynvgcCEsNwVeCvkl9T7/CR5J/U+D/ugS1huB6wI+7HW1wK+CPwiLN8IZDf0/TRR3k8C19fSvtl/T3Tk0Lga5Qp5jZmQp+iV/E6Q94nMBBa4e5m7byM+8eP54Zbv7lvd/SiwAJgZPkFdCiwM21ffB6eb9053XxmWDwIbiF/AKqn3+UnyPpGk2Odhvx0KD9uFm5/ktar/HBYCl4XcTun9NGHeJ9LsvycqDg3nwEtmlmdmc0Ossa6Q19RS+Up+XwqH1U/Y+9chP9W8ewP73b3iuHijCqcsxhP/VJgy+/y4vCHJ97mZtTWzN4hfV2Yp8U/6J3qtf+YX1peE3Jr9b/T4vN392P7+QdjfD5pZ++Pzrmd+p/17ouLQcBe6+wRgOnCHmV18kraNciW8ZtCkV/JrBI8AZwLnATuB/wrxpMvbzLoAzwBfcfcDJ2t6glwSknsteSf9Pnf3Snc/j/jFws4HRp/ktZI2bzMbC9wLjAImET9VdHdo3ux5qzg0kLsXhvsi4Dniv5SNdYW8ppaQK/mdLnffHf6gqoBfEt/nDcl7D/HD8rTj4o3CzNoR/wf7lLs/G8JJv89ryztV9nnIdT+wnPg5+RO91j/zC+u7Ez99mbC/0Wp5Twun99zdy4D/o+H7+/R/T06lg0K3f3YOdQa6Vlv+B/G+gh/zwU7HH4XlK/lgZ9Lr/n5n0jbiHUk9w3KvJsh3CB/s2G20PIlfwGkK73d6zWjCvAdWW/4q8XPEAGfzwc7ErcQ7EtPC8lDe70w8O2zzNB/ssPxiI+VsxM/v/vS4eFLv85PkndT7HOgL9AjLHYG/Aled6LWAO/hgh3ROQ99PE+U9sNrP46fAA4n6PWnUf0Kt5UZ85MLqcFsHfDPEewPLgM3h/tgPyYCHiZ8LXQNEqj3X54h3fuUDtzRBrr8lfjqgnPiniVsbM08gAqwN2/wP4YuVTZT3r0JebxK/JG31f1zfDDlsotqoDOKjPN4K67553M/w9fB+ngbaN1LeFxE/fH8TeCPcZiT7Pj9J3km9z4FzgVUhv7XAv5/stYAO4XF+WD+soe+nifJ+OezvtcCveX9EU7P/nugb0iIiUoP6HEREpAYVBxERqUHFQUREalBxEBGRGlQcRESkBhUHERGpQcVBRERqUHEQEZEa/j+DsjI6sqU3KQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e1fd3a8320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Analysis for choosing optimum word freq cut off\n",
    "freqsL = list(range(1, 80, 5))\n",
    "num_words = []\n",
    "num_sents = []\n",
    "for x in freqsL:\n",
    "    filtered_words = dp.filter_words(word_freqs, x)\n",
    "    num_words.append(len(filtered_words))\n",
    "    num_sents.append(len(dp.get_sentences_with_words(clean_quotes, filtered_words)))\n",
    "plt.plot(num_words, num_sents)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of filtered words =  8961\n",
      "Number of sentences:  43383\n"
     ]
    }
   ],
   "source": [
    "# Considering only words which have a frequency of atleast 15 and the corresponding sentences\n",
    "final_vocabulary = dp.filter_words(word_freqs, 9)\n",
    "print(\"Number of filtered words = \", len(final_vocabulary))\n",
    "final_clean_quotes = dp.get_sentences_with_words(clean_quotes, final_vocabulary)\n",
    "print(\"Number of sentences: \", len(final_clean_quotes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final vocabulary size:  8916\n",
      "Number of sentences:  43383\n",
      "[\"sos don't limit a child to your own learning , for he was born in another time . eos\", \"sos it took me so many years to move out . i'm definitely a bit of a peter pan , reluctant to grow up . it all seemed really nice at home - why change it ? part of me would prefer not to have any responsibility whatsoever . eos\"]\n",
      "['sos', \"don't\", 'limit', 'a', 'child', 'to', 'your', 'own', 'learning', ',']\n"
     ]
    }
   ],
   "source": [
    "# Let's append sos and eos tokens to all sentences\n",
    "clean_quotes_final = [[\"sos\"] + tokens + [\"eos\"] for tokens in final_clean_quotes]\n",
    "clean_quotes_final = [' '.join(tokens) for tokens in clean_quotes_final]\n",
    "vocabulary_final = dp.filter_words(dp.get_vocabulary(dp.tokenize(clean_quotes_final)), 0)\n",
    "print(\"Final vocabulary size: \", len(vocabulary_final))\n",
    "print(\"Number of sentences: \", len(clean_quotes_final))\n",
    "print(clean_quotes_final[:2])\n",
    "print(vocabulary_final[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./datasets/quotes.pickle\",\"wb\") as fop:\n",
    "    pickle.dump(clean_quotes_final, fop)\n",
    "    \n",
    "with open(\"./datasets/quotes_vocabulary.pickle\",\"wb\") as fop:\n",
    "    pickle.dump(vocabulary_final, fop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"sos don't limit a child to your own learning , for he was born in another time . eos\", \"sos it took me so many years to move out . i'm definitely a bit of a peter pan , reluctant to grow up . it all seemed really nice at home - why change it ? part of me would prefer not to have any responsibility whatsoever . eos\", 'sos men take only their needs into consideration - never their abilities . eos', 'sos to walk around with an ego is a bad thing . to have confidence in yourself is a great thing . eos', 'sos an ill wound is cured , not an ill name eos', 'sos i try to become more humble and more myself with every year . there was a while when i got famous where i was so confused and my head was spinning . eos', \"sos i think the industry finally gets it . they've lost the connection with the american public , and they've got to rebuild the trust with the american public . eos\", \"sos i'm so happy to be able to give kids the opportunity to learn about amazing world of dance and music that i've have been lucky enough to make such a big part of my own life . eos\", 'sos in politics , there are no friends . eos', 'sos you only need to make one big score in finance to be a hero forever . eos', 'sos women need real moments of solitude and self - reflection to balance out how much of ourselves we give away . eos', 'sos the wise man does at once what the fool does finally . eos', \"sos i've never intended to be controversial , but it's very easy to be controversial in pop music because nobody ever is . eos\", 'sos i would love to continue acting . maybe one day i would like to try writing and directing . eos', \"sos if you don't want to work you have to work to earn enough money so that you won't have to work . eos\"]\n"
     ]
    }
   ],
   "source": [
    "print(clean_quotes_final[:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}