content_based.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8f3e2bd2",
   "metadata": {},
   "source": [
    "# Content Based Recommender Systems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2c2786ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import*\n",
    "\n",
    "def square_root(x):\n",
    "    return sqrt(sum([a*a for a in x]))\n",
    "\n",
    "def cosine_similarity(x,y):\n",
    "    numerator=sum([x*y for x,y in zip(x,y)])\n",
    "    denominator=square_root(x)*square_root(y)\n",
    "    return round(numerator/denominator,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f40b56a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.707\n"
     ]
    }
   ],
   "source": [
    "print(cosine_similarity([0.5,0.5],[0,0.3]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5dae310",
   "metadata": {},
   "source": [
    "We want to base on our movie data to recommend similar movies based on similar plot, genre, director, etc. For example if we search for Titanic movie, similar movies should be recommended to us, which can be done based on the plots, directors,cast etc..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b5db8dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics.pairwise import cosine_similarity # to perform same work as the cosine similarity we created above\n",
    "from sklearn.feature_extraction.text import CountVectorizer #perform same work as the Document Term Frequency\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6d6a58db",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns',100)\n",
    "df=pd.read_csv('https://query.data.world/s/uikepcpffyo2nhig52xxeevdialfl7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "de23d7b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250, 38)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "62f45265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Title</th>\n",
       "      <th>Year</th>\n",
       "      <th>Rated</th>\n",
       "      <th>Released</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Writer</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Language</th>\n",
       "      <th>Country</th>\n",
       "      <th>Awards</th>\n",
       "      <th>Poster</th>\n",
       "      <th>Ratings.Source</th>\n",
       "      <th>Ratings.Value</th>\n",
       "      <th>Metascore</th>\n",
       "      <th>imdbRating</th>\n",
       "      <th>imdbVotes</th>\n",
       "      <th>imdbID</th>\n",
       "      <th>Type</th>\n",
       "      <th>tomatoMeter</th>\n",
       "      <th>tomatoImage</th>\n",
       "      <th>tomatoRating</th>\n",
       "      <th>tomatoReviews</th>\n",
       "      <th>tomatoFresh</th>\n",
       "      <th>tomatoRotten</th>\n",
       "      <th>tomatoConsensus</th>\n",
       "      <th>tomatoUserMeter</th>\n",
       "      <th>tomatoUserRating</th>\n",
       "      <th>tomatoUserReviews</th>\n",
       "      <th>tomatoURL</th>\n",
       "      <th>DVD</th>\n",
       "      <th>BoxOffice</th>\n",
       "      <th>Production</th>\n",
       "      <th>Website</th>\n",
       "      <th>Response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>1994</td>\n",
       "      <td>R</td>\n",
       "      <td>14 Oct 1994</td>\n",
       "      <td>142 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Frank Darabont</td>\n",
       "      <td>Stephen King (short story \"Rita Hayworth and S...</td>\n",
       "      <td>Tim Robbins, Morgan Freeman, Bob Gunton, Willi...</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "      <td>English</td>\n",
       "      <td>USA</td>\n",
       "      <td>Nominated for 7 Oscars. Another 19 wins &amp; 30 n...</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>Internet Movie Database</td>\n",
       "      <td>9.3/10</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9.3</td>\n",
       "      <td>1,825,626</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>movie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.rottentomatoes.com/m/shawshank_rede...</td>\n",
       "      <td>27 Jan 1998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Columbia Pictures</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>The Godfather</td>\n",
       "      <td>1972</td>\n",
       "      <td>R</td>\n",
       "      <td>24 Mar 1972</td>\n",
       "      <td>175 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Mario Puzo (screenplay), Francis Ford Coppola ...</td>\n",
       "      <td>Marlon Brando, Al Pacino, James Caan, Richard ...</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "      <td>English, Italian, Latin</td>\n",
       "      <td>USA</td>\n",
       "      <td>Won 3 Oscars. Another 23 wins &amp; 27 nominations.</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>Internet Movie Database</td>\n",
       "      <td>9.2/10</td>\n",
       "      <td>100.0</td>\n",
       "      <td>9.2</td>\n",
       "      <td>1,243,444</td>\n",
       "      <td>tt0068646</td>\n",
       "      <td>movie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.rottentomatoes.com/m/godfather/</td>\n",
       "      <td>09 Oct 2001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Paramount Pictures</td>\n",
       "      <td>http://www.thegodfather.com</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>1974</td>\n",
       "      <td>R</td>\n",
       "      <td>20 Dec 1974</td>\n",
       "      <td>202 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Francis Ford Coppola (screenplay), Mario Puzo ...</td>\n",
       "      <td>Al Pacino, Robert Duvall, Diane Keaton, Robert...</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "      <td>English, Italian, Spanish, Latin, Sicilian</td>\n",
       "      <td>USA</td>\n",
       "      <td>Won 6 Oscars. Another 10 wins &amp; 20 nominations.</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>Internet Movie Database</td>\n",
       "      <td>9.0/10</td>\n",
       "      <td>85.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>856,870</td>\n",
       "      <td>tt0071562</td>\n",
       "      <td>movie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.rottentomatoes.com/m/godfather_part...</td>\n",
       "      <td>24 May 2005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Paramount Pictures</td>\n",
       "      <td>http://www.thegodfather.com/</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>2008</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>18 Jul 2008</td>\n",
       "      <td>152 min</td>\n",
       "      <td>Action, Crime, Drama</td>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>Jonathan Nolan (screenplay), Christopher Nolan...</td>\n",
       "      <td>Christian Bale, Heath Ledger, Aaron Eckhart, M...</td>\n",
       "      <td>When the menace known as the Joker emerges fro...</td>\n",
       "      <td>English, Mandarin</td>\n",
       "      <td>USA, UK</td>\n",
       "      <td>Won 2 Oscars. Another 151 wins &amp; 153 nominations.</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>Internet Movie Database</td>\n",
       "      <td>9.0/10</td>\n",
       "      <td>82.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1,802,351</td>\n",
       "      <td>tt0468569</td>\n",
       "      <td>movie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.rottentomatoes.com/m/the_dark_knight/</td>\n",
       "      <td>09 Dec 2008</td>\n",
       "      <td>$533,316,061</td>\n",
       "      <td>Warner Bros. Pictures/Legendary</td>\n",
       "      <td>http://thedarkknight.warnerbros.com/</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>1957</td>\n",
       "      <td>APPROVED</td>\n",
       "      <td>01 Apr 1957</td>\n",
       "      <td>96 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Sidney Lumet</td>\n",
       "      <td>Reginald Rose (story), Reginald Rose (screenplay)</td>\n",
       "      <td>Martin Balsam, John Fiedler, Lee J. Cobb, E.G....</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "      <td>English</td>\n",
       "      <td>USA</td>\n",
       "      <td>Nominated for 3 Oscars. Another 16 wins &amp; 8 no...</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "      <td>Internet Movie Database</td>\n",
       "      <td>8.9/10</td>\n",
       "      <td>96.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>494,215</td>\n",
       "      <td>tt0050083</td>\n",
       "      <td>movie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://www.rottentomatoes.com/m/1000013-12_ang...</td>\n",
       "      <td>06 Mar 2001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Criterion Collection</td>\n",
       "      <td>http://www.criterion.com/films/27871-12-angry-men</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                     Title  Year     Rated     Released  Runtime  \\\n",
       "0           1  The Shawshank Redemption  1994         R  14 Oct 1994  142 min   \n",
       "1           2             The Godfather  1972         R  24 Mar 1972  175 min   \n",
       "2           3    The Godfather: Part II  1974         R  20 Dec 1974  202 min   \n",
       "3           4           The Dark Knight  2008     PG-13  18 Jul 2008  152 min   \n",
       "4           5              12 Angry Men  1957  APPROVED  01 Apr 1957   96 min   \n",
       "\n",
       "                  Genre              Director  \\\n",
       "0          Crime, Drama        Frank Darabont   \n",
       "1          Crime, Drama  Francis Ford Coppola   \n",
       "2          Crime, Drama  Francis Ford Coppola   \n",
       "3  Action, Crime, Drama     Christopher Nolan   \n",
       "4          Crime, Drama          Sidney Lumet   \n",
       "\n",
       "                                              Writer  \\\n",
       "0  Stephen King (short story \"Rita Hayworth and S...   \n",
       "1  Mario Puzo (screenplay), Francis Ford Coppola ...   \n",
       "2  Francis Ford Coppola (screenplay), Mario Puzo ...   \n",
       "3  Jonathan Nolan (screenplay), Christopher Nolan...   \n",
       "4  Reginald Rose (story), Reginald Rose (screenplay)   \n",
       "\n",
       "                                              Actors  \\\n",
       "0  Tim Robbins, Morgan Freeman, Bob Gunton, Willi...   \n",
       "1  Marlon Brando, Al Pacino, James Caan, Richard ...   \n",
       "2  Al Pacino, Robert Duvall, Diane Keaton, Robert...   \n",
       "3  Christian Bale, Heath Ledger, Aaron Eckhart, M...   \n",
       "4  Martin Balsam, John Fiedler, Lee J. Cobb, E.G....   \n",
       "\n",
       "                                                Plot  \\\n",
       "0  Two imprisoned men bond over a number of years...   \n",
       "1  The aging patriarch of an organized crime dyna...   \n",
       "2  The early life and career of Vito Corleone in ...   \n",
       "3  When the menace known as the Joker emerges fro...   \n",
       "4  A jury holdout attempts to prevent a miscarria...   \n",
       "\n",
       "                                     Language  Country  \\\n",
       "0                                     English      USA   \n",
       "1                     English, Italian, Latin      USA   \n",
       "2  English, Italian, Spanish, Latin, Sicilian      USA   \n",
       "3                           English, Mandarin  USA, UK   \n",
       "4                                     English      USA   \n",
       "\n",
       "                                              Awards  \\\n",
       "0  Nominated for 7 Oscars. Another 19 wins & 30 n...   \n",
       "1    Won 3 Oscars. Another 23 wins & 27 nominations.   \n",
       "2    Won 6 Oscars. Another 10 wins & 20 nominations.   \n",
       "3  Won 2 Oscars. Another 151 wins & 153 nominations.   \n",
       "4  Nominated for 3 Oscars. Another 16 wins & 8 no...   \n",
       "\n",
       "                                              Poster           Ratings.Source  \\\n",
       "0  https://images-na.ssl-images-amazon.com/images...  Internet Movie Database   \n",
       "1  https://images-na.ssl-images-amazon.com/images...  Internet Movie Database   \n",
       "2  https://images-na.ssl-images-amazon.com/images...  Internet Movie Database   \n",
       "3  https://images-na.ssl-images-amazon.com/images...  Internet Movie Database   \n",
       "4  https://images-na.ssl-images-amazon.com/images...  Internet Movie Database   \n",
       "\n",
       "  Ratings.Value  Metascore  imdbRating  imdbVotes     imdbID   Type  \\\n",
       "0        9.3/10       80.0         9.3  1,825,626  tt0111161  movie   \n",
       "1        9.2/10      100.0         9.2  1,243,444  tt0068646  movie   \n",
       "2        9.0/10       85.0         9.0    856,870  tt0071562  movie   \n",
       "3        9.0/10       82.0         9.0  1,802,351  tt0468569  movie   \n",
       "4        8.9/10       96.0         8.9    494,215  tt0050083  movie   \n",
       "\n",
       "   tomatoMeter  tomatoImage  tomatoRating  tomatoReviews  tomatoFresh  \\\n",
       "0          NaN          NaN           NaN            NaN          NaN   \n",
       "1          NaN          NaN           NaN            NaN          NaN   \n",
       "2          NaN          NaN           NaN            NaN          NaN   \n",
       "3          NaN          NaN           NaN            NaN          NaN   \n",
       "4          NaN          NaN           NaN            NaN          NaN   \n",
       "\n",
       "   tomatoRotten  tomatoConsensus  tomatoUserMeter  tomatoUserRating  \\\n",
       "0           NaN              NaN              NaN               NaN   \n",
       "1           NaN              NaN              NaN               NaN   \n",
       "2           NaN              NaN              NaN               NaN   \n",
       "3           NaN              NaN              NaN               NaN   \n",
       "4           NaN              NaN              NaN               NaN   \n",
       "\n",
       "   tomatoUserReviews                                          tomatoURL  \\\n",
       "0                NaN  http://www.rottentomatoes.com/m/shawshank_rede...   \n",
       "1                NaN         http://www.rottentomatoes.com/m/godfather/   \n",
       "2                NaN  http://www.rottentomatoes.com/m/godfather_part...   \n",
       "3                NaN   http://www.rottentomatoes.com/m/the_dark_knight/   \n",
       "4                NaN  http://www.rottentomatoes.com/m/1000013-12_ang...   \n",
       "\n",
       "           DVD     BoxOffice                       Production  \\\n",
       "0  27 Jan 1998           NaN                Columbia Pictures   \n",
       "1  09 Oct 2001           NaN               Paramount Pictures   \n",
       "2  24 May 2005           NaN               Paramount Pictures   \n",
       "3  09 Dec 2008  $533,316,061  Warner Bros. Pictures/Legendary   \n",
       "4  06 Mar 2001           NaN             Criterion Collection   \n",
       "\n",
       "                                             Website  Response  \n",
       "0                                                NaN      True  \n",
       "1                        http://www.thegodfather.com      True  \n",
       "2                       http://www.thegodfather.com/      True  \n",
       "3               http://thedarkknight.warnerbros.com/      True  \n",
       "4  http://www.criterion.com/films/27871-12-angry-men      True  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c1df712d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Unnamed: 0', 'Title', 'Year', 'Rated', 'Released', 'Runtime', 'Genre', 'Director', 'Writer', 'Actors', 'Plot', 'Language', 'Country', 'Awards', 'Poster', 'Ratings.Source', 'Ratings.Value', 'Metascore', 'imdbRating', 'imdbVotes', 'imdbID', 'Type', 'tomatoMeter', 'tomatoImage', 'tomatoRating', 'tomatoReviews', 'tomatoFresh', 'tomatoRotten', 'tomatoConsensus', 'tomatoUserMeter', 'tomatoUserRating', 'tomatoUserReviews', 'tomatoURL', 'DVD', 'BoxOffice', 'Production', 'Website', 'Response']\n"
     ]
    }
   ],
   "source": [
    "#for col in df.columns:\n",
    "    #print(col)\n",
    "print(list(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ce38af67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38\n"
     ]
    }
   ],
   "source": [
    "print(len((list(df.columns))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43be3dde",
   "metadata": {},
   "source": [
    "1. there are too many features some are relevant and sum are irrelevant\n",
    "2. In this case we are going to use 'Title','genre','Director', 'Actors','Plot'\n",
    "3. Title='movie name', genre='category(eg;comedy, romance,horror)'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab477c6b",
   "metadata": {},
   "source": [
    "we will base on the 'Title', 'Genre', 'Director','Actors','Plot'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9736f4e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df[['Title','Genre','Director','Actors','Plot']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eb302697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250, 5)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ce99327d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Frank Darabont</td>\n",
       "      <td>Tim Robbins, Morgan Freeman, Bob Gunton, Willi...</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Marlon Brando, Al Pacino, James Caan, Richard ...</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Al Pacino, Robert Duvall, Diane Keaton, Robert...</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>Action, Crime, Drama</td>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>Christian Bale, Heath Ledger, Aaron Eckhart, M...</td>\n",
       "      <td>When the menace known as the Joker emerges fro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Sidney Lumet</td>\n",
       "      <td>Martin Balsam, John Fiedler, Lee J. Cobb, E.G....</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title                 Genre              Director  \\\n",
       "0  The Shawshank Redemption          Crime, Drama        Frank Darabont   \n",
       "1             The Godfather          Crime, Drama  Francis Ford Coppola   \n",
       "2    The Godfather: Part II          Crime, Drama  Francis Ford Coppola   \n",
       "3           The Dark Knight  Action, Crime, Drama     Christopher Nolan   \n",
       "4              12 Angry Men          Crime, Drama          Sidney Lumet   \n",
       "\n",
       "                                              Actors  \\\n",
       "0  Tim Robbins, Morgan Freeman, Bob Gunton, Willi...   \n",
       "1  Marlon Brando, Al Pacino, James Caan, Richard ...   \n",
       "2  Al Pacino, Robert Duvall, Diane Keaton, Robert...   \n",
       "3  Christian Bale, Heath Ledger, Aaron Eckhart, M...   \n",
       "4  Martin Balsam, John Fiedler, Lee J. Cobb, E.G....   \n",
       "\n",
       "                                                Plot  \n",
       "0  Two imprisoned men bond over a number of years...  \n",
       "1  The aging patriarch of an organized crime dyna...  \n",
       "2  The early life and career of Vito Corleone in ...  \n",
       "3  When the menace known as the Joker emerges fro...  \n",
       "4  A jury holdout attempts to prevent a miscarria...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "234d14fe",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#discarding the commas between actors full names  and getting only the first three actors\n",
    "df['Actors']=df['Actors'].map(lambda x: x.split(\",\")[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a854d479",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Frank Darabont</td>\n",
       "      <td>[Tim Robbins,  Morgan Freeman,  Bob Gunton]</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>[Marlon Brando,  Al Pacino,  James Caan]</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>[Al Pacino,  Robert Duvall,  Diane Keaton]</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>Action, Crime, Drama</td>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>[Christian Bale,  Heath Ledger,  Aaron Eckhart]</td>\n",
       "      <td>When the menace known as the Joker emerges fro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Sidney Lumet</td>\n",
       "      <td>[Martin Balsam,  John Fiedler,  Lee J. Cobb]</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title                 Genre              Director  \\\n",
       "0  The Shawshank Redemption          Crime, Drama        Frank Darabont   \n",
       "1             The Godfather          Crime, Drama  Francis Ford Coppola   \n",
       "2    The Godfather: Part II          Crime, Drama  Francis Ford Coppola   \n",
       "3           The Dark Knight  Action, Crime, Drama     Christopher Nolan   \n",
       "4              12 Angry Men          Crime, Drama          Sidney Lumet   \n",
       "\n",
       "                                            Actors  \\\n",
       "0      [Tim Robbins,  Morgan Freeman,  Bob Gunton]   \n",
       "1         [Marlon Brando,  Al Pacino,  James Caan]   \n",
       "2       [Al Pacino,  Robert Duvall,  Diane Keaton]   \n",
       "3  [Christian Bale,  Heath Ledger,  Aaron Eckhart]   \n",
       "4     [Martin Balsam,  John Fiedler,  Lee J. Cobb]   \n",
       "\n",
       "                                                Plot  \n",
       "0  Two imprisoned men bond over a number of years...  \n",
       "1  The aging patriarch of an organized crime dyna...  \n",
       "2  The early life and career of Vito Corleone in ...  \n",
       "3  When the menace known as the Joker emerges fro...  \n",
       "4  A jury holdout attempts to prevent a miscarria...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "98600fa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Genre']=df['Genre'].map(lambda x: x.lower().split(','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "468028d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>Frank Darabont</td>\n",
       "      <td>[Tim Robbins,  Morgan Freeman,  Bob Gunton]</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>[Marlon Brando,  Al Pacino,  James Caan]</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>[Al Pacino,  Robert Duvall,  Diane Keaton]</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>[action,  crime,  drama]</td>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>[Christian Bale,  Heath Ledger,  Aaron Eckhart]</td>\n",
       "      <td>When the menace known as the Joker emerges fro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>Sidney Lumet</td>\n",
       "      <td>[Martin Balsam,  John Fiedler,  Lee J. Cobb]</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title                     Genre              Director  \\\n",
       "0  The Shawshank Redemption           [crime,  drama]        Frank Darabont   \n",
       "1             The Godfather           [crime,  drama]  Francis Ford Coppola   \n",
       "2    The Godfather: Part II           [crime,  drama]  Francis Ford Coppola   \n",
       "3           The Dark Knight  [action,  crime,  drama]     Christopher Nolan   \n",
       "4              12 Angry Men           [crime,  drama]          Sidney Lumet   \n",
       "\n",
       "                                            Actors  \\\n",
       "0      [Tim Robbins,  Morgan Freeman,  Bob Gunton]   \n",
       "1         [Marlon Brando,  Al Pacino,  James Caan]   \n",
       "2       [Al Pacino,  Robert Duvall,  Diane Keaton]   \n",
       "3  [Christian Bale,  Heath Ledger,  Aaron Eckhart]   \n",
       "4     [Martin Balsam,  John Fiedler,  Lee J. Cobb]   \n",
       "\n",
       "                                                Plot  \n",
       "0  Two imprisoned men bond over a number of years...  \n",
       "1  The aging patriarch of an organized crime dyna...  \n",
       "2  The early life and career of Vito Corleone in ...  \n",
       "3  When the menace known as the Joker emerges fro...  \n",
       "4  A jury holdout attempts to prevent a miscarria...  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3894806c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Director']=df['Director'].map(lambda x:x.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "990004d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>[Frank, Darabont]</td>\n",
       "      <td>[Tim Robbins,  Morgan Freeman,  Bob Gunton]</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>[Francis, Ford, Coppola]</td>\n",
       "      <td>[Marlon Brando,  Al Pacino,  James Caan]</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>[Francis, Ford, Coppola]</td>\n",
       "      <td>[Al Pacino,  Robert Duvall,  Diane Keaton]</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>[action,  crime,  drama]</td>\n",
       "      <td>[Christopher, Nolan]</td>\n",
       "      <td>[Christian Bale,  Heath Ledger,  Aaron Eckhart]</td>\n",
       "      <td>When the menace known as the Joker emerges fro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>[Sidney, Lumet]</td>\n",
       "      <td>[Martin Balsam,  John Fiedler,  Lee J. Cobb]</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title                     Genre  \\\n",
       "0  The Shawshank Redemption           [crime,  drama]   \n",
       "1             The Godfather           [crime,  drama]   \n",
       "2    The Godfather: Part II           [crime,  drama]   \n",
       "3           The Dark Knight  [action,  crime,  drama]   \n",
       "4              12 Angry Men           [crime,  drama]   \n",
       "\n",
       "                   Director                                           Actors  \\\n",
       "0         [Frank, Darabont]      [Tim Robbins,  Morgan Freeman,  Bob Gunton]   \n",
       "1  [Francis, Ford, Coppola]         [Marlon Brando,  Al Pacino,  James Caan]   \n",
       "2  [Francis, Ford, Coppola]       [Al Pacino,  Robert Duvall,  Diane Keaton]   \n",
       "3      [Christopher, Nolan]  [Christian Bale,  Heath Ledger,  Aaron Eckhart]   \n",
       "4           [Sidney, Lumet]     [Martin Balsam,  John Fiedler,  Lee J. Cobb]   \n",
       "\n",
       "                                                Plot  \n",
       "0  Two imprisoned men bond over a number of years...  \n",
       "1  The aging patriarch of an organized crime dyna...  \n",
       "2  The early life and career of Vito Corleone in ...  \n",
       "3  When the menace known as the Joker emerges fro...  \n",
       "4  A jury holdout attempts to prevent a miscarria...  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7831c55f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert director and actors names to lower case to avoid duplicates\n",
    "for index, row in df.iterrows():\n",
    "    row['Actors']=[x.lower().replace(' ','') for x in row['Actors']]\n",
    "    row['Director']=''.join(row['Director']).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "07b31767",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                 [timrobbins, morganfreeman, bobgunton]\n",
       "1                    [marlonbrando, alpacino, jamescaan]\n",
       "2                  [alpacino, robertduvall, dianekeaton]\n",
       "3             [christianbale, heathledger, aaroneckhart]\n",
       "4                 [martinbalsam, johnfiedler, leej.cobb]\n",
       "                             ...                        \n",
       "245                [raymilland, janewyman, phillipterry]\n",
       "246     [brielarson, johngallagherjr., stephaniebeatriz]\n",
       "247           [carygrant, rosalindrussell, ralphbellamy]\n",
       "248    [sissyspacek, janegallowayheitz, josepha.carpe...\n",
       "249                [devpatel, saurabhshukla, anilkapoor]\n",
       "Name: Actors, Length: 250, dtype: object"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Actors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "16b55ddd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                  frankdarabont\n",
       "1             francisfordcoppola\n",
       "2             francisfordcoppola\n",
       "3               christophernolan\n",
       "4                    sidneylumet\n",
       "                 ...            \n",
       "245                  billywilder\n",
       "246          destindanielcretton\n",
       "247                  howardhawks\n",
       "248                   davidlynch\n",
       "249    dannyboyle,loveleentandan\n",
       "Name: Director, Length: 250, dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Director']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "61a98661",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting rake_nltkNote: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "  Downloading rake_nltk-1.0.6-py3-none-any.whl (9.1 kB)\n",
      "Requirement already satisfied: nltk<4.0.0,>=3.6.2 in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rake_nltk) (3.8.1)\n",
      "Requirement already satisfied: click in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (8.1.3)\n",
      "Requirement already satisfied: joblib in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (1.2.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (2023.3.23)\n",
      "Requirement already satisfied: tqdm in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (4.65.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\intel\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from click->nltk<4.0.0,>=3.6.2->rake_nltk) (0.4.6)\n",
      "Installing collected packages: rake_nltk\n",
      "Successfully installed rake_nltk-1.0.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 23.1.2 -> 24.0\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "source": [
    "pip install rake_nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "443b7b27",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Intel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Intel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import rake_nltk\n",
    "from rake_nltk import Rake #for extracting significant keywords in the entire text available\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "104d95a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#initializing the new column\n",
    "df['key_words']=\"\"\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    plot=row['Plot']\n",
    "    #instantiating Rake, by default is uses english stopwords from nltk\n",
    "    #and discard all punctuation characters\n",
    "    r=Rake()\n",
    "    \n",
    "    #extracting the words by passing th text\n",
    "    r.extract_keywords_from_text(plot)\n",
    "    \n",
    "    #getting the dictionary with keywords and their scores\n",
    "    key_words_dict_scores=r.get_word_degrees()\n",
    "    \n",
    "    #assigning the key words to the new column\n",
    "    row['key_words']=list(key_words_dict_scores.keys())\n",
    "    \n",
    "#dropping the Plot column\n",
    "df.drop(columns=['Plot'], inplace=True)\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6c3761ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function rake_nltk.rake.Rake._build_word_co_occurance_graph.<locals>.<lambda>()>,\n",
       "            {'mumbai': 3,\n",
       "             'teen': 3,\n",
       "             'reflects': 3,\n",
       "             'upbringing': 1,\n",
       "             'slums': 1,\n",
       "             'accused': 1,\n",
       "             'cheating': 1,\n",
       "             'indian': 2,\n",
       "             'version': 2,\n",
       "             'wants': 1,\n",
       "             'millionaire': 2,\n",
       "             '?\"': 2})"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " key_words_dict_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4d6f5fea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>key_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>frankdarabont</td>\n",
       "      <td>[timrobbins, morganfreeman, bobgunton]</td>\n",
       "      <td>[two, imprisoned, men, bond, number, years, fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>francisfordcoppola</td>\n",
       "      <td>[marlonbrando, alpacino, jamescaan]</td>\n",
       "      <td>[aging, patriarch, organized, crime, dynasty, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>francisfordcoppola</td>\n",
       "      <td>[alpacino, robertduvall, dianekeaton]</td>\n",
       "      <td>[early, life, career, vito, corleone, 1920s, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>[action,  crime,  drama]</td>\n",
       "      <td>christophernolan</td>\n",
       "      <td>[christianbale, heathledger, aaroneckhart]</td>\n",
       "      <td>[menace, known, joker, emerges, mysterious, pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>sidneylumet</td>\n",
       "      <td>[martinbalsam, johnfiedler, leej.cobb]</td>\n",
       "      <td>[jury, holdout, attempts, prevent, miscarriage...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title                     Genre            Director  \\\n",
       "0  The Shawshank Redemption           [crime,  drama]       frankdarabont   \n",
       "1             The Godfather           [crime,  drama]  francisfordcoppola   \n",
       "2    The Godfather: Part II           [crime,  drama]  francisfordcoppola   \n",
       "3           The Dark Knight  [action,  crime,  drama]    christophernolan   \n",
       "4              12 Angry Men           [crime,  drama]         sidneylumet   \n",
       "\n",
       "                                       Actors  \\\n",
       "0      [timrobbins, morganfreeman, bobgunton]   \n",
       "1         [marlonbrando, alpacino, jamescaan]   \n",
       "2       [alpacino, robertduvall, dianekeaton]   \n",
       "3  [christianbale, heathledger, aaroneckhart]   \n",
       "4      [martinbalsam, johnfiedler, leej.cobb]   \n",
       "\n",
       "                                           key_words  \n",
       "0  [two, imprisoned, men, bond, number, years, fi...  \n",
       "1  [aging, patriarch, organized, crime, dynasty, ...  \n",
       "2  [early, life, career, vito, corleone, 1920s, n...  \n",
       "3  [menace, known, joker, emerges, mysterious, pa...  \n",
       "4  [jury, holdout, attempts, prevent, miscarriage...  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "48d33fdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Actors</th>\n",
       "      <th>key_words</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>The Shawshank Redemption</th>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>frankdarabont</td>\n",
       "      <td>[timrobbins, morganfreeman, bobgunton]</td>\n",
       "      <td>[two, imprisoned, men, bond, number, years, fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Godfather</th>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>francisfordcoppola</td>\n",
       "      <td>[marlonbrando, alpacino, jamescaan]</td>\n",
       "      <td>[aging, patriarch, organized, crime, dynasty, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Godfather: Part II</th>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>francisfordcoppola</td>\n",
       "      <td>[alpacino, robertduvall, dianekeaton]</td>\n",
       "      <td>[early, life, career, vito, corleone, 1920s, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Dark Knight</th>\n",
       "      <td>[action,  crime,  drama]</td>\n",
       "      <td>christophernolan</td>\n",
       "      <td>[christianbale, heathledger, aaroneckhart]</td>\n",
       "      <td>[menace, known, joker, emerges, mysterious, pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men</th>\n",
       "      <td>[crime,  drama]</td>\n",
       "      <td>sidneylumet</td>\n",
       "      <td>[martinbalsam, johnfiedler, leej.cobb]</td>\n",
       "      <td>[jury, holdout, attempts, prevent, miscarriage...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Genre            Director  \\\n",
       "Title                                                                    \n",
       "The Shawshank Redemption           [crime,  drama]       frankdarabont   \n",
       "The Godfather                      [crime,  drama]  francisfordcoppola   \n",
       "The Godfather: Part II             [crime,  drama]  francisfordcoppola   \n",
       "The Dark Knight           [action,  crime,  drama]    christophernolan   \n",
       "12 Angry Men                       [crime,  drama]         sidneylumet   \n",
       "\n",
       "                                                              Actors  \\\n",
       "Title                                                                  \n",
       "The Shawshank Redemption      [timrobbins, morganfreeman, bobgunton]   \n",
       "The Godfather                    [marlonbrando, alpacino, jamescaan]   \n",
       "The Godfather: Part II         [alpacino, robertduvall, dianekeaton]   \n",
       "The Dark Knight           [christianbale, heathledger, aaroneckhart]   \n",
       "12 Angry Men                  [martinbalsam, johnfiedler, leej.cobb]   \n",
       "\n",
       "                                                                  key_words  \n",
       "Title                                                                        \n",
       "The Shawshank Redemption  [two, imprisoned, men, bond, number, years, fi...  \n",
       "The Godfather             [aging, patriarch, organized, crime, dynasty, ...  \n",
       "The Godfather: Part II    [early, life, career, vito, corleone, 1920s, n...  \n",
       "The Dark Knight           [menace, known, joker, emerges, mysterious, pa...  \n",
       "12 Angry Men              [jury, holdout, attempts, prevent, miscarriage...  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index('Title', inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "73efd48a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bag_of_words</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Title</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>The Shawshank Redemption</th>\n",
       "      <td>crime  drama frankdarabont timrobbins morganfr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Godfather</th>\n",
       "      <td>crime  drama francisfordcoppola marlonbrando a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Godfather: Part II</th>\n",
       "      <td>crime  drama francisfordcoppola alpacino rober...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Dark Knight</th>\n",
       "      <td>action  crime  drama christophernolan christia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men</th>\n",
       "      <td>crime  drama sidneylumet martinbalsam johnfied...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Lost Weekend</th>\n",
       "      <td>drama  film-noir billywilder raymilland janewy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Short Term 12</th>\n",
       "      <td>drama destindanielcretton brielarson johngalla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>His Girl Friday</th>\n",
       "      <td>comedy  drama  romance howardhawks carygrant r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Straight Story</th>\n",
       "      <td>biography  drama davidlynch sissyspacek janega...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Slumdog Millionaire</th>\n",
       "      <td>drama dannyboyle,loveleentandan devpatel saura...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                               bag_of_words\n",
       "Title                                                                      \n",
       "The Shawshank Redemption  crime  drama frankdarabont timrobbins morganfr...\n",
       "The Godfather             crime  drama francisfordcoppola marlonbrando a...\n",
       "The Godfather: Part II    crime  drama francisfordcoppola alpacino rober...\n",
       "The Dark Knight           action  crime  drama christophernolan christia...\n",
       "12 Angry Men              crime  drama sidneylumet martinbalsam johnfied...\n",
       "...                                                                     ...\n",
       "The Lost Weekend          drama  film-noir billywilder raymilland janewy...\n",
       "Short Term 12             drama destindanielcretton brielarson johngalla...\n",
       "His Girl Friday           comedy  drama  romance howardhawks carygrant r...\n",
       "The Straight Story        biography  drama davidlynch sissyspacek janega...\n",
       "Slumdog Millionaire       drama dannyboyle,loveleentandan devpatel saura...\n",
       "\n",
       "[250 rows x 1 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['bag_of_words']=''\n",
    "columns=df.columns\n",
    "for index, row in df.iterrows():\n",
    "    words=''\n",
    "    for col in columns:\n",
    "        if col != 'Director':\n",
    "            words=words+ ' '.join(row[col]) +' '\n",
    "        else:\n",
    "            words=words +row[col]+ ' '\n",
    "    row['bag_of_words'] =words\n",
    "\n",
    "df.drop(columns =[col for col in df.columns if col!= 'bag_of_words'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "2c281b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#instantiating and generating the count matrix\n",
    "count=CountVectorizer()\n",
    "count_matrix=count.fit_transform(df['bag_of_words'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "bf512673",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<250x2961 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 5342 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a61b6764",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c=count_matrix.todense()\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e1625048",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 584)\t1\n",
      "  (0, 768)\t1\n",
      "  (0, 1011)\t1\n",
      "  (0, 2678)\t1\n",
      "  (0, 1810)\t1\n",
      "  (0, 306)\t1\n",
      "  (0, 2765)\t1\n",
      "  (0, 1269)\t1\n",
      "  (0, 1733)\t1\n",
      "  (0, 311)\t1\n",
      "  (0, 1899)\t1\n",
      "  (0, 2950)\t1\n",
      "  (0, 969)\t1\n",
      "  (0, 2481)\t1\n",
      "  (0, 888)\t1\n",
      "  (0, 2174)\t1\n",
      "  (0, 59)\t1\n",
      "  (0, 519)\t1\n",
      "  (0, 655)\t1\n"
     ]
    }
   ],
   "source": [
    "print(count_matrix[0,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c9c6271b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.15789474, 0.13764944, ..., 0.05263158, 0.05263158,\n",
       "        0.05564149],\n",
       "       [0.15789474, 1.        , 0.36706517, ..., 0.05263158, 0.05263158,\n",
       "        0.05564149],\n",
       "       [0.13764944, 0.36706517, 1.        , ..., 0.04588315, 0.04588315,\n",
       "        0.04850713],\n",
       "       ...,\n",
       "       [0.05263158, 0.05263158, 0.04588315, ..., 1.        , 0.05263158,\n",
       "        0.05564149],\n",
       "       [0.05263158, 0.05263158, 0.04588315, ..., 0.05263158, 1.        ,\n",
       "        0.05564149],\n",
       "       [0.05564149, 0.05564149, 0.04850713, ..., 0.05564149, 0.05564149,\n",
       "        1.        ]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#generating the cosine similarity matrix\n",
    "cosine_sim=cosine_similarity(count_matrix, count_matrix)\n",
    "cosine_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "58f10476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                              The Shawshank Redemption\n",
       "1                                         The Godfather\n",
       "2                                The Godfather: Part II\n",
       "3                                       The Dark Knight\n",
       "4                                          12 Angry Men\n",
       "5                                      Schindler's List\n",
       "6         The Lord of the Rings: The Return of the King\n",
       "7                                          Pulp Fiction\n",
       "8                                            Fight Club\n",
       "9     The Lord of the Rings: The Fellowship of the Ring\n",
       "10                                         Forrest Gump\n",
       "11       Star Wars: Episode V - The Empire Strikes Back\n",
       "12                                            Inception\n",
       "13                The Lord of the Rings: The Two Towers\n",
       "14                      One Flew Over the Cuckoo's Nest\n",
       "15                                           Goodfellas\n",
       "16                                           The Matrix\n",
       "17                   Star Wars: Episode IV - A New Hope\n",
       "18                                                Se7en\n",
       "19                                It's a Wonderful Life\n",
       "Name: Title, dtype: object"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices=pd.Series(df.index)\n",
    "indices[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "37530429",
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommendations(title, cosine_sim=cosine_sim):\n",
    "    recommended_movies=[]\n",
    "    \n",
    "    #getting index of the movie that matches the title\n",
    "    idx=indices[indices==title].index[0]\n",
    "    \n",
    "    #creating Series with the similarity scores in descending order\n",
    "    score_series=pd.Series(cosine_sim[idx]).sort_values(ascending=False)\n",
    "    \n",
    "    #getting the indexes of the 10 most similar movies\n",
    "    top_10_indexes=list(score_series.iloc[1:11].index)\n",
    "    print(top_10_indexes)\n",
    "    \n",
    "    #population the list with the titles of the best 10 matching movies\n",
    "    for i in top_10_indexes:\n",
    "        recommended_movies.append(list(df.index)[i])\n",
    "    return recommended_movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "7071638d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 83, 128, 226, 100, 15, 123, 76, 110, 66]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['The Godfather: Part II',\n",
       " 'Scarface',\n",
       " 'Fargo',\n",
       " 'Rope',\n",
       " 'On the Waterfront',\n",
       " 'Goodfellas',\n",
       " 'Cool Hand Luke',\n",
       " 'Baby Driver',\n",
       " 'Casino',\n",
       " 'A Clockwork Orange']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommendations('The Godfather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "933aeb4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Godfather: Part II'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78be7fd5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}