Assignment 7/Ibrahim Mohamed A7 Recomendation System.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "---\n",
    "# Cairo University Faculty of Engineering\n",
    "## Machine Learning\n",
    "## Assignment 7\n",
    "\n",
    "---\n",
    "Please write your full name here\n",
    "- **Name** : Ibrahim Mohamed"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1. Read the data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "outputs": [],
   "source": [
    "# import the data\n",
    "movies = pd.read_csv('data/movies.csv')\n",
    "ratings = pd.read_csv('data/ratings.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 2. Exploring the data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "outputs": [
    {
     "data": {
      "text/plain": "   movieId                               title  \\\n0        1                    Toy Story (1995)   \n1        2                      Jumanji (1995)   \n2        3             Grumpier Old Men (1995)   \n3        4            Waiting to Exhale (1995)   \n4        5  Father of the Bride Part II (1995)   \n\n                                        genres  \n0  Adventure|Animation|Children|Comedy|Fantasy  \n1                   Adventure|Children|Fantasy  \n2                               Comedy|Romance  \n3                         Comedy|Drama|Romance  \n4                                       Comedy  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movieId</th>\n      <th>title</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>Toy Story (1995)</td>\n      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>Jumanji (1995)</td>\n      <td>Adventure|Children|Fantasy</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>Grumpier Old Men (1995)</td>\n      <td>Comedy|Romance</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4</td>\n      <td>Waiting to Exhale (1995)</td>\n      <td>Comedy|Drama|Romance</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>Father of the Bride Part II (1995)</td>\n      <td>Comedy</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "outputs": [
    {
     "data": {
      "text/plain": "   userId  movieId  rating  timestamp\n0       1        1     4.0  964982703\n1       1        3     4.0  964981247\n2       1        6     4.0  964982224\n3       1       47     5.0  964983815\n4       1       50     5.0  964982931",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>userId</th>\n      <th>movieId</th>\n      <th>rating</th>\n      <th>timestamp</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n      <td>4.0</td>\n      <td>964982703</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>3</td>\n      <td>4.0</td>\n      <td>964981247</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>6</td>\n      <td>4.0</td>\n      <td>964982224</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>47</td>\n      <td>5.0</td>\n      <td>964983815</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>50</td>\n      <td>5.0</td>\n      <td>964982931</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 3. Data Preprocessing"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "outputs": [],
   "source": [
    "# drop the timestamp column\n",
    "ratings.drop('timestamp', axis=1, inplace=True)\n",
    "\n",
    "# drop the genres column\n",
    "movies.drop('genres', axis=1, inplace=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of users =  610\n",
      "Number of movies =  9724\n"
     ]
    }
   ],
   "source": [
    "print('Number of users = ', ratings.userId.nunique())\n",
    "print('Number of movies = ', ratings.movieId.nunique())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "outputs": [],
   "source": [
    "# get the first 200 users\n",
    "ratings = ratings[ratings.userId <= 200]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "outputs": [],
   "source": [
    "# merge the two dataframes\n",
    "movies_ratings = pd.merge(ratings, movies, on='movieId')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "outputs": [
    {
     "data": {
      "text/plain": "   userId  movieId  rating             title\n0       1        1     4.0  Toy Story (1995)\n1       5        1     4.0  Toy Story (1995)\n2       7        1     4.5  Toy Story (1995)\n3      15        1     2.5  Toy Story (1995)\n4      17        1     4.5  Toy Story (1995)",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>userId</th>\n      <th>movieId</th>\n      <th>rating</th>\n      <th>title</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n      <td>4.0</td>\n      <td>Toy Story (1995)</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5</td>\n      <td>1</td>\n      <td>4.0</td>\n      <td>Toy Story (1995)</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7</td>\n      <td>1</td>\n      <td>4.5</td>\n      <td>Toy Story (1995)</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>15</td>\n      <td>1</td>\n      <td>2.5</td>\n      <td>Toy Story (1995)</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>17</td>\n      <td>1</td>\n      <td>4.5</td>\n      <td>Toy Story (1995)</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_ratings.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "We need to take care as these ratings are not accurate because some movies have few ratings so we need to make sure that we are not considering movies with few ratings\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "outputs": [],
   "source": [
    "# Get the number of reviews given to each movie\n",
    "top_rated = movies_ratings.groupby('movieId').size().reset_index(name='rating_count')\n",
    "\n",
    "# Filter movies with over 20 ratings in descending order\n",
    "top_rated = top_rated[top_rated['rating_count'] > 20]\n",
    "top_rated.sort_values('rating_count', ascending=False, inplace=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "outputs": [],
   "source": [
    "movies_ratings = movies_ratings[movies_ratings['movieId'].isin(top_rated['movieId'])]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of users =  198\n",
      "Number of movies =  287\n"
     ]
    }
   ],
   "source": [
    "print('Number of users = ', movies_ratings.userId.nunique())\n",
    "print('Number of movies = ', movies_ratings.movieId.nunique())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Get the Similarity Score between the users & movies\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "outputs": [
    {
     "data": {
      "text/plain": "movieId  1       2       6       7       10      11      16      17      \\\nuserId                                                                    \n1           4.0     NaN     4.0     NaN     NaN     NaN     NaN     NaN   \n2           NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n3           NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n4           NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n5           4.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n\nmovieId  19      21      ...  59315   60069   68157   68954   69122   72998   \\\nuserId                   ...                                                   \n1           NaN     NaN  ...     NaN     NaN     NaN     NaN     NaN     NaN   \n2           NaN     NaN  ...     NaN     NaN     4.5     NaN     NaN     NaN   \n3           NaN     NaN  ...     NaN     NaN     NaN     NaN     NaN     NaN   \n4           NaN     3.0  ...     NaN     NaN     NaN     NaN     NaN     NaN   \n5           NaN     4.0  ...     NaN     NaN     NaN     NaN     NaN     NaN   \n\nmovieId  79132   91529   99114   109487  \nuserId                                   \n1           NaN     NaN     NaN     NaN  \n2           4.0     3.5     3.5     3.0  \n3           NaN     NaN     NaN     NaN  \n4           NaN     NaN     NaN     NaN  \n5           NaN     NaN     NaN     NaN  \n\n[5 rows x 287 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>movieId</th>\n      <th>1</th>\n      <th>2</th>\n      <th>6</th>\n      <th>7</th>\n      <th>10</th>\n      <th>11</th>\n      <th>16</th>\n      <th>17</th>\n      <th>19</th>\n      <th>21</th>\n      <th>...</th>\n      <th>59315</th>\n      <th>60069</th>\n      <th>68157</th>\n      <th>68954</th>\n      <th>69122</th>\n      <th>72998</th>\n      <th>79132</th>\n      <th>91529</th>\n      <th>99114</th>\n      <th>109487</th>\n    </tr>\n    <tr>\n      <th>userId</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>3.5</td>\n      <td>3.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>3.0</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 287 columns</p>\n</div>"
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a pivot table to get the ratings of each user for each movie\n",
    "pivot_table = movies_ratings.pivot_table(index='userId', columns='movieId', values='rating')\n",
    "pivot_table.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "outputs": [],
   "source": [
    "# fill the missing values with 0\n",
    "pivot_table = pivot_table.fillna(0)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# get the cosine similarity between the users\n",
    "user_cosine_similarity = cosine_similarity(pivot_table)\n",
    "\n",
    "# get the cosine similarity between the movies\n",
    "movies_cosine_similarity = cosine_similarity(pivot_table.T)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "outputs": [
    {
     "data": {
      "text/plain": "userId       1         2         3         4         5         6         7    \\\nuserId                                                                         \n1       1.000000  0.037354  0.085398  0.445768  0.248501  0.272933  0.283794   \n2       0.037354  1.000000  0.000000  0.011914  0.032843  0.032521  0.023597   \n3       0.085398  0.000000  1.000000  0.000000  0.166852  0.059479  0.000000   \n4       0.445768  0.011914  0.000000  1.000000  0.143811  0.166084  0.251298   \n5       0.248501  0.032843  0.166852  0.143811  1.000000  0.541198  0.146181   \n6       0.272933  0.032521  0.059479  0.166084  0.541198  1.000000  0.169422   \n7       0.283794  0.023597  0.000000  0.251298  0.146181  0.169422  1.000000   \n8       0.231540  0.047312  0.144217  0.124302  0.523611  0.633620  0.156227   \n9       0.163338  0.000000  0.000000  0.033719  0.000000  0.000000  0.203362   \n10      0.057914  0.217943  0.000000  0.103929  0.075837  0.076525  0.325378   \n\nuserId       8         9         10   ...       191       192       193  \\\nuserId                                ...                                 \n1       0.231540  0.163338  0.057914  ...  0.183733  0.177892  0.264730   \n2       0.047312  0.000000  0.217943  ...  0.042264  0.000000  0.041582   \n3       0.144217  0.000000  0.000000  ...  0.000000  0.000000  0.144857   \n4       0.124302  0.033719  0.103929  ...  0.109169  0.044500  0.234100   \n5       0.523611  0.000000  0.075837  ...  0.395522  0.253008  0.141393   \n6       0.633620  0.000000  0.076525  ...  0.418897  0.498027  0.109853   \n7       0.156227  0.203362  0.325378  ...  0.081317  0.169393  0.282111   \n8       1.000000  0.000000  0.052023  ...  0.358589  0.488178  0.176528   \n9       0.000000  1.000000  0.133970  ...  0.059808  0.000000  0.084060   \n10      0.052023  0.133970  1.000000  ...  0.011618  0.056633  0.090138   \n\nuserId       194       195       196       197       198       199       200  \nuserId                                                                        \n1       0.107127  0.417716  0.165295  0.237922  0.507726  0.464626  0.354420  \n2       0.000000  0.111246  0.213271  0.047852  0.085147  0.103454  0.155851  \n3       0.000000  0.083045  0.000000  0.000000  0.069213  0.089250  0.000000  \n4       0.105946  0.407147  0.082855  0.130666  0.413061  0.471924  0.318739  \n5       0.000000  0.199531  0.142444  0.206872  0.221728  0.246605  0.194297  \n6       0.107003  0.209103  0.163423  0.143152  0.269644  0.273918  0.253480  \n7       0.000000  0.267463  0.112758  0.160529  0.342770  0.340121  0.487154  \n8       0.000000  0.188033  0.140103  0.193534  0.259524  0.247131  0.235028  \n9       0.000000  0.071323  0.051249  0.000000  0.207247  0.126786  0.200431  \n10      0.000000  0.134805  0.202429  0.093396  0.150426  0.103668  0.366869  \n\n[10 rows x 198 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>userId</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n      <th>...</th>\n      <th>191</th>\n      <th>192</th>\n      <th>193</th>\n      <th>194</th>\n      <th>195</th>\n      <th>196</th>\n      <th>197</th>\n      <th>198</th>\n      <th>199</th>\n      <th>200</th>\n    </tr>\n    <tr>\n      <th>userId</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>1.000000</td>\n      <td>0.037354</td>\n      <td>0.085398</td>\n      <td>0.445768</td>\n      <td>0.248501</td>\n      <td>0.272933</td>\n      <td>0.283794</td>\n      <td>0.231540</td>\n      <td>0.163338</td>\n      <td>0.057914</td>\n      <td>...</td>\n      <td>0.183733</td>\n      <td>0.177892</td>\n      <td>0.264730</td>\n      <td>0.107127</td>\n      <td>0.417716</td>\n      <td>0.165295</td>\n      <td>0.237922</td>\n      <td>0.507726</td>\n      <td>0.464626</td>\n      <td>0.354420</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.037354</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>0.011914</td>\n      <td>0.032843</td>\n      <td>0.032521</td>\n      <td>0.023597</td>\n      <td>0.047312</td>\n      <td>0.000000</td>\n      <td>0.217943</td>\n      <td>...</td>\n      <td>0.042264</td>\n      <td>0.000000</td>\n      <td>0.041582</td>\n      <td>0.000000</td>\n      <td>0.111246</td>\n      <td>0.213271</td>\n      <td>0.047852</td>\n      <td>0.085147</td>\n      <td>0.103454</td>\n      <td>0.155851</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.085398</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>0.166852</td>\n      <td>0.059479</td>\n      <td>0.000000</td>\n      <td>0.144217</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.144857</td>\n      <td>0.000000</td>\n      <td>0.083045</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.069213</td>\n      <td>0.089250</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.445768</td>\n      <td>0.011914</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.143811</td>\n      <td>0.166084</td>\n      <td>0.251298</td>\n      <td>0.124302</td>\n      <td>0.033719</td>\n      <td>0.103929</td>\n      <td>...</td>\n      <td>0.109169</td>\n      <td>0.044500</td>\n      <td>0.234100</td>\n      <td>0.105946</td>\n      <td>0.407147</td>\n      <td>0.082855</td>\n      <td>0.130666</td>\n      <td>0.413061</td>\n      <td>0.471924</td>\n      <td>0.318739</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.248501</td>\n      <td>0.032843</td>\n      <td>0.166852</td>\n      <td>0.143811</td>\n      <td>1.000000</td>\n      <td>0.541198</td>\n      <td>0.146181</td>\n      <td>0.523611</td>\n      <td>0.000000</td>\n      <td>0.075837</td>\n      <td>...</td>\n      <td>0.395522</td>\n      <td>0.253008</td>\n      <td>0.141393</td>\n      <td>0.000000</td>\n      <td>0.199531</td>\n      <td>0.142444</td>\n      <td>0.206872</td>\n      <td>0.221728</td>\n      <td>0.246605</td>\n      <td>0.194297</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.272933</td>\n      <td>0.032521</td>\n      <td>0.059479</td>\n      <td>0.166084</td>\n      <td>0.541198</td>\n      <td>1.000000</td>\n      <td>0.169422</td>\n      <td>0.633620</td>\n      <td>0.000000</td>\n      <td>0.076525</td>\n      <td>...</td>\n      <td>0.418897</td>\n      <td>0.498027</td>\n      <td>0.109853</td>\n      <td>0.107003</td>\n      <td>0.209103</td>\n      <td>0.163423</td>\n      <td>0.143152</td>\n      <td>0.269644</td>\n      <td>0.273918</td>\n      <td>0.253480</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.283794</td>\n      <td>0.023597</td>\n      <td>0.000000</td>\n      <td>0.251298</td>\n      <td>0.146181</td>\n      <td>0.169422</td>\n      <td>1.000000</td>\n      <td>0.156227</td>\n      <td>0.203362</td>\n      <td>0.325378</td>\n      <td>...</td>\n      <td>0.081317</td>\n      <td>0.169393</td>\n      <td>0.282111</td>\n      <td>0.000000</td>\n      <td>0.267463</td>\n      <td>0.112758</td>\n      <td>0.160529</td>\n      <td>0.342770</td>\n      <td>0.340121</td>\n      <td>0.487154</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.231540</td>\n      <td>0.047312</td>\n      <td>0.144217</td>\n      <td>0.124302</td>\n      <td>0.523611</td>\n      <td>0.633620</td>\n      <td>0.156227</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>0.052023</td>\n      <td>...</td>\n      <td>0.358589</td>\n      <td>0.488178</td>\n      <td>0.176528</td>\n      <td>0.000000</td>\n      <td>0.188033</td>\n      <td>0.140103</td>\n      <td>0.193534</td>\n      <td>0.259524</td>\n      <td>0.247131</td>\n      <td>0.235028</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.163338</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.033719</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.203362</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.133970</td>\n      <td>...</td>\n      <td>0.059808</td>\n      <td>0.000000</td>\n      <td>0.084060</td>\n      <td>0.000000</td>\n      <td>0.071323</td>\n      <td>0.051249</td>\n      <td>0.000000</td>\n      <td>0.207247</td>\n      <td>0.126786</td>\n      <td>0.200431</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.057914</td>\n      <td>0.217943</td>\n      <td>0.000000</td>\n      <td>0.103929</td>\n      <td>0.075837</td>\n      <td>0.076525</td>\n      <td>0.325378</td>\n      <td>0.052023</td>\n      <td>0.133970</td>\n      <td>1.000000</td>\n      <td>...</td>\n      <td>0.011618</td>\n      <td>0.056633</td>\n      <td>0.090138</td>\n      <td>0.000000</td>\n      <td>0.134805</td>\n      <td>0.202429</td>\n      <td>0.093396</td>\n      <td>0.150426</td>\n      <td>0.103668</td>\n      <td>0.366869</td>\n    </tr>\n  </tbody>\n</table>\n<p>10 rows × 198 columns</p>\n</div>"
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert the cosine similarity to a dataframe to see the similarity scores between the users\n",
    "user_cosine_similarity_df = pd.DataFrame(user_cosine_similarity, index=pivot_table.index, columns=pivot_table.index)\n",
    "movies_cosine_similarity_df = pd.DataFrame(movies_cosine_similarity, index=pivot_table.columns, columns=pivot_table.columns)\n",
    "\n",
    "user_cosine_similarity_df.head(10)\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "outputs": [
    {
     "data": {
      "text/plain": "movieId    1         2         6         7         10        11        16      \\\nmovieId                                                                         \n1        1.000000  0.364634  0.327928  0.270701  0.352899  0.245691  0.240085   \n2        0.364634  1.000000  0.278936  0.240488  0.342590  0.334386  0.216116   \n6        0.327928  0.278936  1.000000  0.275939  0.304328  0.358692  0.500971   \n7        0.270701  0.240488  0.275939  1.000000  0.263707  0.428225  0.140414   \n10       0.352899  0.342590  0.304328  0.263707  1.000000  0.345764  0.269904   \n11       0.245691  0.334386  0.358692  0.428225  0.345764  1.000000  0.311140   \n16       0.240085  0.216116  0.500971  0.140414  0.269904  0.311140  1.000000   \n17       0.309381  0.144009  0.277037  0.293515  0.292537  0.324717  0.302576   \n19       0.383177  0.344587  0.281046  0.253973  0.352139  0.337001  0.395251   \n21       0.314843  0.219876  0.438093  0.348902  0.219544  0.529989  0.255459   \n\nmovieId    17        19        21      ...    59315     60069     68157   \\\nmovieId                                ...                                 \n1        0.309381  0.383177  0.314843  ...  0.234873  0.199627  0.210979   \n2        0.144009  0.344587  0.219876  ...  0.234106  0.214452  0.224016   \n6        0.277037  0.281046  0.438093  ...  0.191202  0.163585  0.220964   \n7        0.293515  0.253973  0.348902  ...  0.040530  0.027472  0.037330   \n10       0.292537  0.352139  0.219544  ...  0.141756  0.106430  0.078231   \n11       0.324717  0.337001  0.529989  ...  0.081979  0.050823  0.075507   \n16       0.302576  0.395251  0.255459  ...  0.225522  0.203236  0.345011   \n17       1.000000  0.237369  0.407341  ...  0.047536  0.049013  0.079227   \n19       0.237369  1.000000  0.341195  ...  0.203139  0.171669  0.171437   \n21       0.407341  0.341195  1.000000  ...  0.036531  0.030569  0.022431   \n\nmovieId    68954     69122     72998     79132     91529     99114     109487  \nmovieId                                                                        \n1        0.246652  0.228514  0.228129  0.244432  0.250658  0.242432  0.209214  \n2        0.307064  0.286514  0.321419  0.254434  0.268707  0.336075  0.262833  \n6        0.152767  0.193568  0.226812  0.192702  0.231025  0.190655  0.170448  \n7        0.024964  0.070391  0.036230  0.032816  0.033671  0.026482  0.000000  \n10       0.144320  0.143295  0.149755  0.105132  0.138895  0.100751  0.031529  \n11       0.052342  0.143986  0.071493  0.067455  0.069213  0.055717  0.000000  \n16       0.217168  0.276929  0.265345  0.274850  0.273005  0.280906  0.191939  \n17       0.053447  0.050340  0.052610  0.058547  0.043096  0.031176  0.006769  \n19       0.238604  0.241137  0.240706  0.198263  0.276363  0.295057  0.239871  \n21       0.016667  0.028656  0.055634  0.016432  0.023417  0.020801  0.024275  \n\n[10 rows x 287 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>movieId</th>\n      <th>1</th>\n      <th>2</th>\n      <th>6</th>\n      <th>7</th>\n      <th>10</th>\n      <th>11</th>\n      <th>16</th>\n      <th>17</th>\n      <th>19</th>\n      <th>21</th>\n      <th>...</th>\n      <th>59315</th>\n      <th>60069</th>\n      <th>68157</th>\n      <th>68954</th>\n      <th>69122</th>\n      <th>72998</th>\n      <th>79132</th>\n      <th>91529</th>\n      <th>99114</th>\n      <th>109487</th>\n    </tr>\n    <tr>\n      <th>movieId</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>1.000000</td>\n      <td>0.364634</td>\n      <td>0.327928</td>\n      <td>0.270701</td>\n      <td>0.352899</td>\n      <td>0.245691</td>\n      <td>0.240085</td>\n      <td>0.309381</td>\n      <td>0.383177</td>\n      <td>0.314843</td>\n      <td>...</td>\n      <td>0.234873</td>\n      <td>0.199627</td>\n      <td>0.210979</td>\n      <td>0.246652</td>\n      <td>0.228514</td>\n      <td>0.228129</td>\n      <td>0.244432</td>\n      <td>0.250658</td>\n      <td>0.242432</td>\n      <td>0.209214</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.364634</td>\n      <td>1.000000</td>\n      <td>0.278936</td>\n      <td>0.240488</td>\n      <td>0.342590</td>\n      <td>0.334386</td>\n      <td>0.216116</td>\n      <td>0.144009</td>\n      <td>0.344587</td>\n      <td>0.219876</td>\n      <td>...</td>\n      <td>0.234106</td>\n      <td>0.214452</td>\n      <td>0.224016</td>\n      <td>0.307064</td>\n      <td>0.286514</td>\n      <td>0.321419</td>\n      <td>0.254434</td>\n      <td>0.268707</td>\n      <td>0.336075</td>\n      <td>0.262833</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.327928</td>\n      <td>0.278936</td>\n      <td>1.000000</td>\n      <td>0.275939</td>\n      <td>0.304328</td>\n      <td>0.358692</td>\n      <td>0.500971</td>\n      <td>0.277037</td>\n      <td>0.281046</td>\n      <td>0.438093</td>\n      <td>...</td>\n      <td>0.191202</td>\n      <td>0.163585</td>\n      <td>0.220964</td>\n      <td>0.152767</td>\n      <td>0.193568</td>\n      <td>0.226812</td>\n      <td>0.192702</td>\n      <td>0.231025</td>\n      <td>0.190655</td>\n      <td>0.170448</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.270701</td>\n      <td>0.240488</td>\n      <td>0.275939</td>\n      <td>1.000000</td>\n      <td>0.263707</td>\n      <td>0.428225</td>\n      <td>0.140414</td>\n      <td>0.293515</td>\n      <td>0.253973</td>\n      <td>0.348902</td>\n      <td>...</td>\n      <td>0.040530</td>\n      <td>0.027472</td>\n      <td>0.037330</td>\n      <td>0.024964</td>\n      <td>0.070391</td>\n      <td>0.036230</td>\n      <td>0.032816</td>\n      <td>0.033671</td>\n      <td>0.026482</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.352899</td>\n      <td>0.342590</td>\n      <td>0.304328</td>\n      <td>0.263707</td>\n      <td>1.000000</td>\n      <td>0.345764</td>\n      <td>0.269904</td>\n      <td>0.292537</td>\n      <td>0.352139</td>\n      <td>0.219544</td>\n      <td>...</td>\n      <td>0.141756</td>\n      <td>0.106430</td>\n      <td>0.078231</td>\n      <td>0.144320</td>\n      <td>0.143295</td>\n      <td>0.149755</td>\n      <td>0.105132</td>\n      <td>0.138895</td>\n      <td>0.100751</td>\n      <td>0.031529</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.245691</td>\n      <td>0.334386</td>\n      <td>0.358692</td>\n      <td>0.428225</td>\n      <td>0.345764</td>\n      <td>1.000000</td>\n      <td>0.311140</td>\n      <td>0.324717</td>\n      <td>0.337001</td>\n      <td>0.529989</td>\n      <td>...</td>\n      <td>0.081979</td>\n      <td>0.050823</td>\n      <td>0.075507</td>\n      <td>0.052342</td>\n      <td>0.143986</td>\n      <td>0.071493</td>\n      <td>0.067455</td>\n      <td>0.069213</td>\n      <td>0.055717</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>0.240085</td>\n      <td>0.216116</td>\n      <td>0.500971</td>\n      <td>0.140414</td>\n      <td>0.269904</td>\n      <td>0.311140</td>\n      <td>1.000000</td>\n      <td>0.302576</td>\n      <td>0.395251</td>\n      <td>0.255459</td>\n      <td>...</td>\n      <td>0.225522</td>\n      <td>0.203236</td>\n      <td>0.345011</td>\n      <td>0.217168</td>\n      <td>0.276929</td>\n      <td>0.265345</td>\n      <td>0.274850</td>\n      <td>0.273005</td>\n      <td>0.280906</td>\n      <td>0.191939</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>0.309381</td>\n      <td>0.144009</td>\n      <td>0.277037</td>\n      <td>0.293515</td>\n      <td>0.292537</td>\n      <td>0.324717</td>\n      <td>0.302576</td>\n      <td>1.000000</td>\n      <td>0.237369</td>\n      <td>0.407341</td>\n      <td>...</td>\n      <td>0.047536</td>\n      <td>0.049013</td>\n      <td>0.079227</td>\n      <td>0.053447</td>\n      <td>0.050340</td>\n      <td>0.052610</td>\n      <td>0.058547</td>\n      <td>0.043096</td>\n      <td>0.031176</td>\n      <td>0.006769</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>0.383177</td>\n      <td>0.344587</td>\n      <td>0.281046</td>\n      <td>0.253973</td>\n      <td>0.352139</td>\n      <td>0.337001</td>\n      <td>0.395251</td>\n      <td>0.237369</td>\n      <td>1.000000</td>\n      <td>0.341195</td>\n      <td>...</td>\n      <td>0.203139</td>\n      <td>0.171669</td>\n      <td>0.171437</td>\n      <td>0.238604</td>\n      <td>0.241137</td>\n      <td>0.240706</td>\n      <td>0.198263</td>\n      <td>0.276363</td>\n      <td>0.295057</td>\n      <td>0.239871</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>0.314843</td>\n      <td>0.219876</td>\n      <td>0.438093</td>\n      <td>0.348902</td>\n      <td>0.219544</td>\n      <td>0.529989</td>\n      <td>0.255459</td>\n      <td>0.407341</td>\n      <td>0.341195</td>\n      <td>1.000000</td>\n      <td>...</td>\n      <td>0.036531</td>\n      <td>0.030569</td>\n      <td>0.022431</td>\n      <td>0.016667</td>\n      <td>0.028656</td>\n      <td>0.055634</td>\n      <td>0.016432</td>\n      <td>0.023417</td>\n      <td>0.020801</td>\n      <td>0.024275</td>\n    </tr>\n  </tbody>\n</table>\n<p>10 rows × 287 columns</p>\n</div>"
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_cosine_similarity_df.head(10)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### TOP SIMILAR MOVIES\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "outputs": [],
   "source": [
    "def topSimilarMovies(movie_id, number_of_similar_movies):\n",
    "\n",
    "    # check if the movie_id exists in the selected movies\n",
    "    if movie_id not in movies_cosine_similarity_df.columns:\n",
    "        return 'Movie not found in the database'\n",
    "    else:\n",
    "        # get the top similar movies\n",
    "        top_movies_idx = movies_cosine_similarity_df[movie_id].sort_values(ascending=False).head(number_of_similar_movies + 1)\n",
    "\n",
    "        # get the titles of the top similar movies\n",
    "        top_movies = []\n",
    "        for item in top_movies_idx.index:\n",
    "            movie = movies_ratings[movies_ratings['movieId'] == item]['title'].values[0]\n",
    "            top_movies.append(movie)\n",
    "\n",
    "        # remove the first movie as it is the same movie\n",
    "        top_movies.pop(0)\n",
    "        return top_movies"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The  selected movie is Dark Knight Rises, The (2012) and the top 10 similar movies are: \n",
      "Dark Knight, The (2008)\n",
      "Up (2009)\n",
      "Inception (2010)\n",
      "Django Unchained (2012)\n",
      "WALL·E (2008)\n",
      "Inglourious Basterds (2009)\n",
      "Interstellar (2014)\n",
      "Iron Man (2008)\n",
      "Batman Begins (2005)\n",
      "Hangover, The (2009)\n"
     ]
    }
   ],
   "source": [
    "# get the move titles of movie_id = 91529\n",
    "my_movie_id = 91529\n",
    "testing_movie = movies_ratings[movies_ratings['movieId'] == my_movie_id]['title'].tolist()[0]\n",
    "num_of_movies = 10\n",
    "\n",
    "# print the top similar moviesA\n",
    "print(f'The  selected movie is {testing_movie} and the top {num_of_movies} similar movies are: ')\n",
    "for item in topSimilarMovies(my_movie_id, num_of_movies):\n",
    "    print(item)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### RECOMMENDATION SYSTEM BASED ON USER"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "outputs": [],
   "source": [
    "def recommendMovieToUser(userId, number_of_similar_movies):\n",
    "\n",
    "    if userId not in user_cosine_similarity_df.index:\n",
    "        return 'User not found in the database'\n",
    "\n",
    "    # get the movies that the user has watched\n",
    "    user_movies = movies_ratings[movies_ratings['userId'] == userId]['movieId'].tolist()\n",
    "\n",
    "    # get the similarity scores between the user and all other users\n",
    "    user_similarity = user_cosine_similarity_df[userId].sort_values(ascending=False)\n",
    "\n",
    "    # get the top 10 similar users\n",
    "    top_similar_users = user_similarity.head(10).index.tolist()\n",
    "\n",
    "    # get the movies that the top 10 similar users have watched\n",
    "    top_similar_users_movies = movies_ratings[movies_ratings['userId'].isin(top_similar_users)]['movieId'].tolist()\n",
    "\n",
    "    # get the movies that the top 10 similar users have watched but the user hasn't\n",
    "    recommended_movies = list(set(top_similar_users_movies) - set(user_movies))\n",
    "\n",
    "    # get the titles of the recommended movies\n",
    "    recommended_movies_titles = []\n",
    "    for movie_id in recommended_movies:\n",
    "        movie_title = movies_ratings[movies_ratings['movieId'] == movie_id]['title'].tolist()[0]\n",
    "        recommended_movies_titles.append(movie_title)\n",
    "\n",
    "    # return the top 5 recommended movies\n",
    "    return recommended_movies_titles[:number_of_similar_movies]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The top 5 recommended movies for user_id = 200 are: \n",
      "Jumanji (1995)\n",
      "Hangover, The (2009)\n",
      "Heat (1995)\n",
      "Sabrina (1995)\n",
      "Robin Hood: Men in Tights (1993)\n"
     ]
    }
   ],
   "source": [
    "# get the top 10 recommended movies for user_id = 200\n",
    "user_id = 200\n",
    "num_of_movies = 5\n",
    "print(f'The top {num_of_movies} recommended movies for user_id = {user_id} are: ')\n",
    "for item in recommendMovieToUser(user_id, num_of_movies):\n",
    "    print(item)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}