3. Organise_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import io\n",
    "import sys\n",
    "import os.path\n",
    "import urllib.request\n",
    "from tqdm import tqdm\n",
    "from os import listdir\n",
    "from PIL import Image\n",
    "import glob\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "np.set_printoptions(threshold=sys.maxsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Paths</th>\n",
       "      <th>imdbId</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Posters/27428.jpg</td>\n",
       "      <td>27428</td>\n",
       "      <td>Crime|Mystery|Drama</td>\n",
       "      <td>The Case of the Black Cat (1936)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Posters/972555.jpg</td>\n",
       "      <td>972555</td>\n",
       "      <td>Crime|Drama|Thriller</td>\n",
       "      <td>Exhibit A (2007)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Posters/1424062.jpg</td>\n",
       "      <td>1424062</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>The Thorn in the Heart (2009)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Posters/1185371.jpg</td>\n",
       "      <td>1185371</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>Corman's World: Exploits of a Hollywood Rebel (2011)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Posters/113369.jpg</td>\n",
       "      <td>113369</td>\n",
       "      <td>Drama</td>\n",
       "      <td>I Shot a Man in Vegas (1995)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Image_Paths   imdbId                 Genre  \\\n",
       "0  Posters/27428.jpg    27428    Crime|Mystery|Drama    \n",
       "1  Posters/972555.jpg   972555   Crime|Drama|Thriller   \n",
       "2  Posters/1424062.jpg  1424062  Documentary            \n",
       "3  Posters/1185371.jpg  1185371  Documentary            \n",
       "4  Posters/113369.jpg   113369   Drama                  \n",
       "\n",
       "                                                  Title  \n",
       "0  The Case of the Black Cat (1936)                      \n",
       "1  Exhibit A (2007)                                      \n",
       "2  The Thorn in the Heart (2009)                         \n",
       "3  Corman's World: Exploits of a Hollywood Rebel (2011)  \n",
       "4  I Shot a Man in Vegas (1995)                          "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"MovieGenre_final.csv\",encoding='ISO-8859-1')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36918, 4)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36918\n"
     ]
    }
   ],
   "source": [
    "# There exists a genre value for all images\n",
    "Genre_list = df['Genre']\n",
    "print(len(Genre_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Breaks \"Genre\" into the constituting individual genres\n",
    "def find_genres(genre):\n",
    "    \n",
    "    start = 0\n",
    "    set_of_genre = []\n",
    "    for i in range(len(genre)):\n",
    "        \n",
    "        k=0\n",
    "        substring = \"\"\n",
    "        if (genre[i]=='|'):\n",
    "            substring = genre[start:i]\n",
    "            start = i+1\n",
    "            k = 1\n",
    "        \n",
    "        if(i==len(genre)-1):\n",
    "            substring = genre[start:i+1]\n",
    "            k = 1\n",
    "            \n",
    "        if (k==1):\n",
    "            set_of_genre.append(substring)         \n",
    "    \n",
    "    return (set_of_genre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique genres: 28\n",
      "Unique genres are: ['Action' 'Adult' 'Adventure' 'Animation' 'Biography' 'Comedy' 'Crime'\n",
      " 'Documentary' 'Drama' 'Family' 'Fantasy' 'Film-Noir' 'Game-Show'\n",
      " 'History' 'Horror' 'Music' 'Musical' 'Mystery' 'News' 'Reality-TV'\n",
      " 'Romance' 'Sci-Fi' 'Short' 'Sport' 'Talk-Show' 'Thriller' 'War' 'Western']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Action': 4960,\n",
       " 'Adult': 8,\n",
       " 'Adventure': 3563,\n",
       " 'Animation': 1609,\n",
       " 'Biography': 1852,\n",
       " 'Comedy': 11779,\n",
       " 'Crime': 4874,\n",
       " 'Documentary': 3451,\n",
       " 'Drama': 18632,\n",
       " 'Family': 1949,\n",
       " 'Fantasy': 1888,\n",
       " 'Film-Noir': 359,\n",
       " 'Game-Show': 1,\n",
       " 'History': 1331,\n",
       " 'Horror': 3751,\n",
       " 'Music': 1194,\n",
       " 'Musical': 762,\n",
       " 'Mystery': 2229,\n",
       " 'News': 78,\n",
       " 'Reality-TV': 2,\n",
       " 'Romance': 5742,\n",
       " 'Sci-Fi': 1866,\n",
       " 'Short': 872,\n",
       " 'Sport': 650,\n",
       " 'Talk-Show': 6,\n",
       " 'Thriller': 4512,\n",
       " 'War': 1092,\n",
       " 'Western': 783}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract list of genre values for each image\n",
    "all_genre = []\n",
    "Genre_list = df['Genre']\n",
    "\n",
    "for i in range (len(Genre_list)):\n",
    "    \n",
    "    set_of_genre = find_genres(Genre_list[i])\n",
    "    \n",
    "    for j in range (len(set_of_genre)):\n",
    "        all_genre.append(set_of_genre[j])\n",
    "        \n",
    "uniq, counts = np.unique(all_genre, return_counts=True)\n",
    "print(\"Number of unique genres:\", len(uniq))\n",
    "print(\"Unique genres are:\", uniq)\n",
    "dict(zip(uniq, counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare multi-hot-encoded-labels for the various genres\n",
    "def multi_hot_encoded_labels(img_id, genre):\n",
    "    \n",
    "    col_names =  ['Action', 'Adult', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime',\n",
    "                  'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Game-Show', 'History',\n",
    "                  'Horror', 'Music', 'Musical', 'Mystery', 'News', 'Reality-TV', 'Romance',\n",
    "                  'Sci-Fi', 'Short', 'Sport', 'Talk-Show', 'Thriller', 'War', 'Western']\n",
    "    \n",
    "    set_of_genre = find_genres(genre)\n",
    "    \n",
    "    row=[]\n",
    "    row.append(img_id)\n",
    "    \n",
    "    for i in range(len(col_names)):\n",
    "        \n",
    "        found = 0\n",
    "        for j in range (len(set_of_genre)):\n",
    "            if (set_of_genre[j]==col_names[i]):\n",
    "                found = 1\n",
    "                break\n",
    "        \n",
    "        row.append(found)\n",
    "    \n",
    "    row.append(genre) #add the overall combined genre for record purposes\n",
    "        \n",
    "    return row  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36918it [00:04, 7589.71it/s]\n"
     ]
    }
   ],
   "source": [
    "# Perform the encoding of the labels and save data in the format :\n",
    "# Img-ID <multi-hot-encoded-labels> overall_genre\n",
    "\n",
    "df = pd.read_csv(\"MovieGenre_final.csv\", encoding=\"ISO-8859-1\")\n",
    "all_data = []\n",
    "\n",
    "for index, row in tqdm(df.iterrows()):\n",
    "    \n",
    "    path = row['Image_Paths']\n",
    "    genre = row['Genre']\n",
    "    row = multi_hot_encoded_labels(path, genre)\n",
    "    \n",
    "    all_data.append(row)\n",
    "\n",
    "col_names =  ['Img-paths', 'Action', 'Adult', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime',\n",
    "                  'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Game-Show', 'History',\n",
    "                  'Horror', 'Music', 'Musical', 'Mystery', 'News', 'Reality-TV', 'Romance',\n",
    "                  'Sci-Fi', 'Short', 'Sport', 'Talk-Show', 'Thriller', 'War', 'Western', 'Genre']\n",
    "\n",
    "np.savetxt(\"Multi_hot_encoded_data.csv\", np.asarray(all_data), fmt='%s', delimiter=\" \")   \n",
    "np.savetxt(\"Encoded_data_column_lookup.csv\", np.asarray(col_names), fmt='%s', delimiter=\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Img-paths</th>\n",
       "      <th>Action</th>\n",
       "      <th>Adult</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Biography</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Documentary</th>\n",
       "      <th>Drama</th>\n",
       "      <th>...</th>\n",
       "      <th>Reality-TV</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Sci-Fi</th>\n",
       "      <th>Short</th>\n",
       "      <th>Sport</th>\n",
       "      <th>Talk-Show</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Posters/27428.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Crime|Mystery|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Posters/972555.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Posters/1424062.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Documentary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Posters/1185371.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Documentary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Posters/113369.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Img-paths  Action  Adult  Adventure  Animation  Biography  \\\n",
       "0  Posters/27428.jpg    0       0      0          0          0           \n",
       "1  Posters/972555.jpg   0       0      0          0          0           \n",
       "2  Posters/1424062.jpg  0       0      0          0          0           \n",
       "3  Posters/1185371.jpg  0       0      0          0          0           \n",
       "4  Posters/113369.jpg   0       0      0          0          0           \n",
       "\n",
       "   Comedy  Crime  Documentary  Drama  ...  Reality-TV  Romance  Sci-Fi  Short  \\\n",
       "0  0       1      0            1      ...  0           0        0       0       \n",
       "1  0       1      0            1      ...  0           0        0       0       \n",
       "2  0       0      1            0      ...  0           0        0       0       \n",
       "3  0       0      1            0      ...  0           0        0       0       \n",
       "4  0       0      0            1      ...  0           0        0       0       \n",
       "\n",
       "   Sport  Talk-Show  Thriller  War  Western                 Genre  \n",
       "0  0      0          0         0    0        Crime|Mystery|Drama   \n",
       "1  0      0          1         0    0        Crime|Drama|Thriller  \n",
       "2  0      0          0         0    0        Documentary           \n",
       "3  0      0          0         0    0        Documentary           \n",
       "4  0      0          0         0    0        Drama                 \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_encoded = pd.read_csv(\"Multi_hot_encoded_data.csv\", delimiter=\" \", \n",
    "                  names =  ['Img-paths', 'Action', 'Adult', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime',\n",
    "                  'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Game-Show', 'History',\n",
    "                  'Horror', 'Music', 'Musical', 'Mystery', 'News', 'Reality-TV', 'Romance',\n",
    "                  'Sci-Fi', 'Short', 'Sport', 'Talk-Show', 'Thriller', 'War', 'Western', 'Genre'])\n",
    "\n",
    "df_encoded.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train_df= 25842\n",
      "Val_df= 9967\n",
      "Test_df= 1108\n"
     ]
    }
   ],
   "source": [
    "# To split into train / validation / test in the ratio 80 / 15 / 5%\n",
    "\n",
    "df = pd.read_csv(\"Multi_hot_encoded_data.csv\", delimiter=\" \")\n",
    "random_seed = 50\n",
    "train_df = df.sample(frac=0.70, random_state=random_seed) #Taking 70% of the data\n",
    "tmp_df = df.drop(train_df.index)\n",
    "test_df = tmp_df.sample(frac=0.1, random_state=random_seed) #Taking 20% of the remaining (after train is taken)\n",
    "valid_df = tmp_df.drop(test_df.index)\n",
    "\n",
    "print(\"Train_df=\",len(train_df))\n",
    "print(\"Val_df=\",len(valid_df))\n",
    "print(\"Test_df=\",len(test_df))\n",
    "\n",
    "np.savetxt(\"Train.csv\", train_df, fmt='%s', delimiter=\" \")\n",
    "np.savetxt(\"Test.csv\", test_df, fmt='%s', delimiter=\" \")\n",
    "np.savetxt(\"Valid.csv\", valid_df, fmt='%s', delimiter=\" \")\n",
    "\n",
    "#Numpy method\n",
    "#train, validate, test = np.split(df_encoded.sample(frac=1), [int(.6*len(df)), int(.8*len(df))])\n",
    "#np.split will split at 60% of the length of the shuffled array, \n",
    "#then 80% of length (which is an additional 20% of data), thus leaving a remaining 20% of the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}