0_Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For text preprocessing\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from string import digits\n",
    "\n",
    "# Generics\n",
    "import pandas as pd\n",
    "import json\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "stemmer = nltk.stem.PorterStemmer()\n",
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_word(word):\n",
    "    \"\"\"\n",
    "    INPUT : String\n",
    "    OUTPUT: String lemmatized\n",
    "    Apply standard cleaning operations of strings like:\n",
    "    - Lemmatization\n",
    "    - Stemming\n",
    "    \"\"\"\n",
    "    word = lemmatizer.lemmatize(word)\n",
    "    return stemmer.stem(word)\n",
    "\n",
    "def preprocess_doc(document):\n",
    "    \n",
    "    \"\"\"\n",
    "    INPUT : String\n",
    "    OUTPUT: List of strings \n",
    "    Apply standard cleaning operations to each of the words\n",
    "    and return a list of the cleaned words\n",
    "    \"\"\"\n",
    "    document = document.lower().translate(str.maketrans(' ', ' ', string.punctuation)).split()\n",
    "    return [preprocess_word(x)  for x in document if x not in stop_words]\n",
    "\n",
    "def get_first_genre(s):\n",
    "    \"\"\"\n",
    "    INPUT : String\n",
    "    OUTPUT: List of strings \n",
    "    \n",
    "    Returns the first element of the list\n",
    "    \"\"\"\n",
    "    return s[0]\n",
    "\n",
    "\n",
    "def parse_string_to_json(s):\n",
    "    \"\"\"\n",
    "    INPUT : A string that resembles a JSON doc(or a dictionary)\n",
    "    OUTPUT: A list of the values of the dictionary\n",
    "    \n",
    "    Parse a string into a JSON and then return the values as a list\n",
    "    \"\"\"\n",
    "    r = list(json.loads(s).values())\n",
    "    if not r:\n",
    "        return ['unknown']\n",
    "    return r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Read the dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Files paths\n",
    "filename          = './data/movies/plot_summaries.txt'\n",
    "movie_names_file  = './data/movies/movie.metadata.tsv'\n",
    "\n",
    "\n",
    "# Set correctly the headers of the dataset\n",
    "movie_headers   = ['Wiki_ID', 'Freebase_ID', 'Name', 'Release_Date','box_office','runtime','languages','countries','genres']\n",
    "summary_headers = ['Wiki_ID', 'Plot']\n",
    "\n",
    "\n",
    "# Read the 2 DFS\n",
    "df_filename = pd.read_csv(filename, delimiter='\\t', header=None, names=summary_headers)\n",
    "df_movie    = pd.read_csv(movie_names_file, delimiter='\\t', header=None,names=movie_headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Merge the 2 dataframes on the Wiki_ID attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Shlykov, a hard-working taxi driver and Lyosha...</td>\n",
       "      <td>Taxi Blues</td>\n",
       "      <td>{\"/m/07s9rl0\": \"Drama\", \"/m/03q4nz\": \"World ci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>{\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>{\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>{\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>{\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        0  Shlykov, a hard-working taxi driver and Lyosha...   \n",
       "1        1  The nation of Panem consists of a wealthy Capi...   \n",
       "2        2  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "3        3  The Lemon Drop Kid , a New York City swindler,...   \n",
       "4        4  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "\n",
       "                 Name                                             genres  \n",
       "0          Taxi Blues  {\"/m/07s9rl0\": \"Drama\", \"/m/03q4nz\": \"World ci...  \n",
       "1    The Hunger Games  {\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...  \n",
       "2          Narasimham  {\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...  \n",
       "3  The Lemon Drop Kid  {\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...  \n",
       "4   A Cry in the Dark  {\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.merge(df_filename,df_movie, on='Wiki_ID')\n",
    "# Drop the columns\n",
    "cols_to_drop = [ 'Freebase_ID', 'Release_Date', 'box_office',  'runtime', 'languages', 'countries']\n",
    "df = df.drop(cols_to_drop, axis=1)\n",
    "\n",
    "# Re-index with consecutive numbers\n",
    "df['Wiki_ID'] = list(range(df.shape[0]))\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove movies that have a too short plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>{\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...</td>\n",
       "      <td>4559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>{\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...</td>\n",
       "      <td>3099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>{\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...</td>\n",
       "      <td>4917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>{\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...</td>\n",
       "      <td>2425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>{\"/m/01jfsb\": \"Thriller\", \"/m/03btsm8\": \"Actio...</td>\n",
       "      <td>1937</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        1  The nation of Panem consists of a wealthy Capi...   \n",
       "1        2  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2        3  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3        4  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4        5  The president is on his way to give a speech. ...   \n",
       "\n",
       "                 Name                                             genres   len  \n",
       "0    The Hunger Games  {\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...  4559  \n",
       "1          Narasimham  {\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...  3099  \n",
       "2  The Lemon Drop Kid  {\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...  4917  \n",
       "3   A Cry in the Dark  {\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...  2425  \n",
       "4            End Game  {\"/m/01jfsb\": \"Thriller\", \"/m/03btsm8\": \"Actio...  1937  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['len'] = df['Plot'].apply(len)\n",
    "\n",
    "# Calculate quantiles (with .3 we will keep docs with more than 3000 words)\n",
    "lower_length_bound = int(df['len'].quantile([.5]).to_list()[0])\n",
    "\n",
    "# Drop the lower bounds of documents\n",
    "df = df.drop(df[df.len < lower_length_bound].index)  \n",
    "\n",
    "df = df.reset_index(drop=True)\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Wiki_ID'] = np.arange(df.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Apply preprocessing to the movies plot in order to obtain a clean corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "      <th>Corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>{\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...</td>\n",
       "      <td>4559</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>{\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...</td>\n",
       "      <td>3099</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>{\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...</td>\n",
       "      <td>4917</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>{\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...</td>\n",
       "      <td>2425</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>{\"/m/01jfsb\": \"Thriller\", \"/m/03btsm8\": \"Actio...</td>\n",
       "      <td>1937</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        0  The nation of Panem consists of a wealthy Capi...   \n",
       "1        1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2        2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3        3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4        4  The president is on his way to give a speech. ...   \n",
       "\n",
       "                 Name                                             genres  \\\n",
       "0    The Hunger Games  {\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...   \n",
       "1          Narasimham  {\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...   \n",
       "2  The Lemon Drop Kid  {\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...   \n",
       "3   A Cry in the Dark  {\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...   \n",
       "4            End Game  {\"/m/01jfsb\": \"Thriller\", \"/m/03btsm8\": \"Actio...   \n",
       "\n",
       "    len                                             Corpus  \n",
       "0  4559  [nation, panem, consist, wealthi, capitol, twe...  \n",
       "1  3099  [pooval, induchoodan, sentenc, six, year, pris...  \n",
       "2  4917  [lemon, drop, kid, new, york, citi, swindler, ...  \n",
       "3  2425  [seventhday, adventist, church, pastor, michae...  \n",
       "4  1937  [presid, way, give, speech, travel, man, show,...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Corpus'] = df['Plot'].apply(preprocess_doc)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Fix the genres attribute by parsing it to JSON and extract the various genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "      <th>Corpus</th>\n",
       "      <th>genres_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>Action/Adventure</td>\n",
       "      <td>4559</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "      <td>[Action/Adventure, Science Fiction, Action, Dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>Musical</td>\n",
       "      <td>3099</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "      <td>[Musical, Action, Drama, Bollywood]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>Screwball comedy</td>\n",
       "      <td>4917</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "      <td>[Screwball comedy, Comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>Crime Fiction</td>\n",
       "      <td>2425</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "      <td>[Crime Fiction, Drama, Docudrama, World cinema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>1937</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "      <td>[Thriller, Action/Adventure, Action, Drama]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        0  The nation of Panem consists of a wealthy Capi...   \n",
       "1        1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2        2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3        3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4        4  The president is on his way to give a speech. ...   \n",
       "\n",
       "                 Name            genres   len  \\\n",
       "0    The Hunger Games  Action/Adventure  4559   \n",
       "1          Narasimham           Musical  3099   \n",
       "2  The Lemon Drop Kid  Screwball comedy  4917   \n",
       "3   A Cry in the Dark     Crime Fiction  2425   \n",
       "4            End Game          Thriller  1937   \n",
       "\n",
       "                                              Corpus  \\\n",
       "0  [nation, panem, consist, wealthi, capitol, twe...   \n",
       "1  [pooval, induchoodan, sentenc, six, year, pris...   \n",
       "2  [lemon, drop, kid, new, york, citi, swindler, ...   \n",
       "3  [seventhday, adventist, church, pastor, michae...   \n",
       "4  [presid, way, give, speech, travel, man, show,...   \n",
       "\n",
       "                                          genres_all  \n",
       "0  [Action/Adventure, Science Fiction, Action, Dr...  \n",
       "1                [Musical, Action, Drama, Bollywood]  \n",
       "2                         [Screwball comedy, Comedy]  \n",
       "3  [Crime Fiction, Drama, Docudrama, World cinema...  \n",
       "4        [Thriller, Action/Adventure, Action, Drama]  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['genres_all']   = df['genres'].apply(parse_string_to_json)\n",
    "df['genres']       = df['genres_all'].apply(get_first_genre)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "      <th>Corpus</th>\n",
       "      <th>genres_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>Action/Adventure</td>\n",
       "      <td>4559</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "      <td>[Action/Adventure, Science Fiction, Action, Dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>Musical</td>\n",
       "      <td>3099</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "      <td>[Musical, Action, Drama, Bollywood]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>Screwball comedy</td>\n",
       "      <td>4917</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "      <td>[Screwball comedy, Comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>Crime Fiction</td>\n",
       "      <td>2425</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "      <td>[Crime Fiction, Drama, Docudrama, World cinema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>1937</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "      <td>[Thriller, Action/Adventure, Action, Drama]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        0  The nation of Panem consists of a wealthy Capi...   \n",
       "1        1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2        2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3        3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4        4  The president is on his way to give a speech. ...   \n",
       "\n",
       "                 Name            genres   len  \\\n",
       "0    The Hunger Games  Action/Adventure  4559   \n",
       "1          Narasimham           Musical  3099   \n",
       "2  The Lemon Drop Kid  Screwball comedy  4917   \n",
       "3   A Cry in the Dark     Crime Fiction  2425   \n",
       "4            End Game          Thriller  1937   \n",
       "\n",
       "                                              Corpus  \\\n",
       "0  [nation, panem, consist, wealthi, capitol, twe...   \n",
       "1  [pooval, induchoodan, sentenc, six, year, pris...   \n",
       "2  [lemon, drop, kid, new, york, citi, swindler, ...   \n",
       "3  [seventhday, adventist, church, pastor, michae...   \n",
       "4  [presid, way, give, speech, travel, man, show,...   \n",
       "\n",
       "                                          genres_all  \n",
       "0  [Action/Adventure, Science Fiction, Action, Dr...  \n",
       "1                [Musical, Action, Drama, Bollywood]  \n",
       "2                         [Screwball comedy, Comedy]  \n",
       "3  [Crime Fiction, Drama, Docudrama, World cinema...  \n",
       "4        [Thriller, Action/Adventure, Action, Drama]  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>Corpus</th>\n",
       "      <th>genres_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>Action/Adventure</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "      <td>[Action/Adventure, Science Fiction, Action, Dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>Musical</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "      <td>[Musical, Action, Drama, Bollywood]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>Screwball comedy</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "      <td>[Screwball comedy, Comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>Crime Fiction</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "      <td>[Crime Fiction, Drama, Docudrama, World cinema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "      <td>[Thriller, Action/Adventure, Action, Drama]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21101</th>\n",
       "      <td>21101</td>\n",
       "      <td>Lucy  is working as a dancer in a sleazy strip...</td>\n",
       "      <td>I Don't Want to Be Born</td>\n",
       "      <td>Horror</td>\n",
       "      <td>[luci, work, dancer, sleazi, strip, joint, sta...</td>\n",
       "      <td>[Horror]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21102</th>\n",
       "      <td>21102</td>\n",
       "      <td>Twenty-something Eun-mo listens to a taxi driv...</td>\n",
       "      <td>Paju</td>\n",
       "      <td>Romantic drama</td>\n",
       "      <td>[twentysometh, eunmo, listen, taxi, driver, dr...</td>\n",
       "      <td>[Romantic drama, Romance Film, Drama, World ci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21103</th>\n",
       "      <td>21103</td>\n",
       "      <td>In 1928 Hollywood, director Leo Andreyev  look...</td>\n",
       "      <td>The Last Command</td>\n",
       "      <td>Silent film</td>\n",
       "      <td>[1928, hollywood, director, leo, andreyev, loo...</td>\n",
       "      <td>[Silent film, Indie, Black-and-white, Period p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21104</th>\n",
       "      <td>21104</td>\n",
       "      <td>Abdur Rehman Khan , a middle-aged dry fruit se...</td>\n",
       "      <td>Kabuliwala</td>\n",
       "      <td>Drama</td>\n",
       "      <td>[abdur, rehman, khan, middleag, dri, fruit, se...</td>\n",
       "      <td>[Drama]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21105</th>\n",
       "      <td>21105</td>\n",
       "      <td>1940 - Operation Dynamo has just taken place. ...</td>\n",
       "      <td>Dad's Army</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>[1940, oper, dynamo, taken, place, newli, conq...</td>\n",
       "      <td>[Comedy]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21106 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Wiki_ID                                               Plot  \\\n",
       "0            0  The nation of Panem consists of a wealthy Capi...   \n",
       "1            1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2            2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3            3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4            4  The president is on his way to give a speech. ...   \n",
       "...        ...                                                ...   \n",
       "21101    21101  Lucy  is working as a dancer in a sleazy strip...   \n",
       "21102    21102  Twenty-something Eun-mo listens to a taxi driv...   \n",
       "21103    21103  In 1928 Hollywood, director Leo Andreyev  look...   \n",
       "21104    21104  Abdur Rehman Khan , a middle-aged dry fruit se...   \n",
       "21105    21105  1940 - Operation Dynamo has just taken place. ...   \n",
       "\n",
       "                          Name            genres  \\\n",
       "0             The Hunger Games  Action/Adventure   \n",
       "1                   Narasimham           Musical   \n",
       "2           The Lemon Drop Kid  Screwball comedy   \n",
       "3            A Cry in the Dark     Crime Fiction   \n",
       "4                     End Game          Thriller   \n",
       "...                        ...               ...   \n",
       "21101  I Don't Want to Be Born            Horror   \n",
       "21102                     Paju    Romantic drama   \n",
       "21103         The Last Command       Silent film   \n",
       "21104               Kabuliwala             Drama   \n",
       "21105               Dad's Army            Comedy   \n",
       "\n",
       "                                                  Corpus  \\\n",
       "0      [nation, panem, consist, wealthi, capitol, twe...   \n",
       "1      [pooval, induchoodan, sentenc, six, year, pris...   \n",
       "2      [lemon, drop, kid, new, york, citi, swindler, ...   \n",
       "3      [seventhday, adventist, church, pastor, michae...   \n",
       "4      [presid, way, give, speech, travel, man, show,...   \n",
       "...                                                  ...   \n",
       "21101  [luci, work, dancer, sleazi, strip, joint, sta...   \n",
       "21102  [twentysometh, eunmo, listen, taxi, driver, dr...   \n",
       "21103  [1928, hollywood, director, leo, andreyev, loo...   \n",
       "21104  [abdur, rehman, khan, middleag, dri, fruit, se...   \n",
       "21105  [1940, oper, dynamo, taken, place, newli, conq...   \n",
       "\n",
       "                                              genres_all  \n",
       "0      [Action/Adventure, Science Fiction, Action, Dr...  \n",
       "1                    [Musical, Action, Drama, Bollywood]  \n",
       "2                             [Screwball comedy, Comedy]  \n",
       "3      [Crime Fiction, Drama, Docudrama, World cinema...  \n",
       "4            [Thriller, Action/Adventure, Action, Drama]  \n",
       "...                                                  ...  \n",
       "21101                                           [Horror]  \n",
       "21102  [Romantic drama, Romance Film, Drama, World ci...  \n",
       "21103  [Silent film, Indie, Black-and-white, Period p...  \n",
       "21104                                            [Drama]  \n",
       "21105                                           [Comedy]  \n",
       "\n",
       "[21106 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop(columns=['len'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Serialize the dataframe into a pickle object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "path   = \"./preprocessed/\"\n",
    "df.to_pickle(path+\"df_preprocessed.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Deep Learning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}