10TrainTheModelForEmbeddings.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "10TrainTheModelForEmbeddings.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CzFF5Z7Kgisi",
        "colab_type": "text"
      },
      "source": [
        "# 1) Import the libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_wGP0kxgmBW",
        "colab_type": "code",
        "outputId": "76271138-15aa-48ac-fc0f-f42ada4c0a5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aicGSc5zgYyC",
        "colab_type": "code",
        "outputId": "92bd01c5-aefa-4a31-d945-654956055374",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-ejWWXhgY01",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import nltk\n",
        "import pandas as pd\n",
        "import gensim\n",
        "from gensim.models import Word2Vec, KeyedVectors"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SggbQKTdjRFt",
        "colab_type": "text"
      },
      "source": [
        "# 2) Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1FKc5vzkiTy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# google pretrained model\n",
        "\n",
        "# https://www.kaggle.com/rootuser/worldnews-on-reddit"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6vanM71UkiYi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# install kaggle API\n",
        "\n",
        "! pip install -q kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "srBxS7p3kiay",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create a directory as kaggle\n",
        "! mkdir -p ~/.kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F4ULh9oakiWw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# copy API key to kaggle directory\n",
        "! cp kaggle.json ~/.kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W7YPYkoWk5ix",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# disable the API key\n",
        "! chmod 600 /root/.kaggle/kaggle.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7fZno58tk5lb",
        "colab_type": "code",
        "outputId": "79612088-f413-4746-be3c-09ab0198da97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "# import the dataset\n",
        "! kaggle datasets download -d rootuser/worldnews-on-reddit"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading worldnews-on-reddit.zip to /content\n",
            " 19% 5.00M/26.6M [00:00<00:00, 23.7MB/s]\n",
            "100% 26.6M/26.6M [00:00<00:00, 88.8MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwcNqEVrk5n8",
        "colab_type": "code",
        "outputId": "05eb5bd3-bf30-4be3-b3b0-1d5103e386ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "# unzip the dataset\n",
        "! unzip /content/worldnews-on-reddit"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/worldnews-on-reddit.zip\n",
            "  inflating: reddit_worldnews_start_to_2016-11-22.csv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gY83YUvVjTeg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('reddit_worldnews_start_to_2016-11-22.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YtEo_mCjTg9",
        "colab_type": "code",
        "outputId": "d4aa6b9d-5bf7-41d1-d7cd-7bf50ed4f5c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_created</th>\n",
              "      <th>date_created</th>\n",
              "      <th>up_votes</th>\n",
              "      <th>down_votes</th>\n",
              "      <th>title</th>\n",
              "      <th>over_18</th>\n",
              "      <th>author</th>\n",
              "      <th>subreddit</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1201232046</td>\n",
              "      <td>2008-01-25</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>Scores killed in Pakistan clashes</td>\n",
              "      <td>False</td>\n",
              "      <td>polar</td>\n",
              "      <td>worldnews</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1201232075</td>\n",
              "      <td>2008-01-25</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>Japan resumes refuelling mission</td>\n",
              "      <td>False</td>\n",
              "      <td>polar</td>\n",
              "      <td>worldnews</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1201232523</td>\n",
              "      <td>2008-01-25</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>US presses Egypt on Gaza border</td>\n",
              "      <td>False</td>\n",
              "      <td>polar</td>\n",
              "      <td>worldnews</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1201233290</td>\n",
              "      <td>2008-01-25</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Jump-start economy: Give health care to all</td>\n",
              "      <td>False</td>\n",
              "      <td>fadi420</td>\n",
              "      <td>worldnews</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1201274720</td>\n",
              "      <td>2008-01-25</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>Council of Europe bashes EU&amp;UN terror blacklist</td>\n",
              "      <td>False</td>\n",
              "      <td>mhermans</td>\n",
              "      <td>worldnews</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   time_created date_created  up_votes  ...  over_18    author  subreddit\n",
              "0    1201232046   2008-01-25         3  ...    False     polar  worldnews\n",
              "1    1201232075   2008-01-25         2  ...    False     polar  worldnews\n",
              "2    1201232523   2008-01-25         3  ...    False     polar  worldnews\n",
              "3    1201233290   2008-01-25         1  ...    False   fadi420  worldnews\n",
              "4    1201274720   2008-01-25         4  ...    False  mhermans  worldnews\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cu2q7hsujTji",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# take title column\n",
        "news_titles = df['title'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZ38kZFWjfOB",
        "colab_type": "code",
        "outputId": "6bc3efe1-6587-4f86-d18b-53bdcdaa1aae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "news_titles"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Scores killed in Pakistan clashes',\n",
              "       'Japan resumes refuelling mission',\n",
              "       'US presses Egypt on Gaza border', ...,\n",
              "       'Professor receives Arab Researchers Award',\n",
              "       'Nigel Farage attacks response to Trump ambassador tweet',\n",
              "       'Palestinian wielding knife shot dead in West Bank: Israel police'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1qx-DeNjfQV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tokenize the words\n",
        "new_vec = [nltk.word_tokenize(title) for title in news_titles]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AutPMAQyjwyx",
        "colab_type": "text"
      },
      "source": [
        "# 3) Build the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQnvwndSjfS4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Word2Vec(new_vec, min_count=1, size=32)\n",
        "# text, min word count, size of each vector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2WXGrVtOj1yw",
        "colab_type": "text"
      },
      "source": [
        "# 4) Predict the output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-i0IMO09jju4",
        "colab_type": "code",
        "outputId": "fb039c5a-a1af-4a74-b3cb-0dcbe92b0b75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# find 10 closest words in the vector space that we have created\n",
        "model.wv.most_similar('man')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('woman', 0.9691265821456909),\n",
              " ('girl', 0.9091819524765015),\n",
              " ('couple', 0.9074012041091919),\n",
              " ('boy', 0.8982519507408142),\n",
              " ('doctor', 0.8895249366760254),\n",
              " ('teenager', 0.8770707845687866),\n",
              " ('mother', 0.8724143505096436),\n",
              " ('teacher', 0.865044355392456),\n",
              " ('family', 0.8362488746643066),\n",
              " ('father', 0.8352950811386108)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zgoOVCsumGBY",
        "colab_type": "code",
        "outputId": "bb89e858-d97f-4e57-f574-99221e04cdd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "# see the vector\n",
        "model.wv['man']\n",
        "\n",
        "# so this is how man is represented in our vector space"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1.5322248 , -0.11720701, -0.14232874, -3.0922418 , -2.544124  ,\n",
              "       -2.4784405 , -4.716915  , -1.3238051 , -5.8409886 ,  3.611334  ,\n",
              "        5.6475005 ,  1.7071834 ,  0.40946892,  0.435135  , -1.1746104 ,\n",
              "       -0.7665485 , -4.5962305 , -4.0302453 , -1.8472059 ,  2.192009  ,\n",
              "       -3.3843486 ,  2.3493419 ,  1.5161991 , -1.9585158 ,  1.1967756 ,\n",
              "       -0.6227615 , -1.5623277 ,  1.1481372 ,  2.4485068 , -0.09623834,\n",
              "       -0.5554248 , -2.042582  ], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5YZL1K-jlPY",
        "colab_type": "code",
        "outputId": "1b4218db-f3f7-4e58-a9ea-47deebc6025b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# let us try the famous relationship\n",
        "vec = model.wv['king'] - model.wv['man'] + model.wv['women']\n",
        "model.wv.most_similar([vec])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('freedoms', 0.6844619512557983),\n",
              " ('women', 0.6701126098632812),\n",
              " ('clerics', 0.6689168214797974),\n",
              " ('religious', 0.667762041091919),\n",
              " ('unions', 0.6629506349563599),\n",
              " ('poor.The', 0.6593372225761414),\n",
              " ('equal', 0.6441993713378906),\n",
              " ('Ethiopian-Israelis', 0.637859046459198),\n",
              " ('CDU/CSU', 0.6360814571380615),\n",
              " ('parties', 0.6271290183067322)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nGHIulpHjn6I",
        "colab_type": "code",
        "outputId": "9302ae3f-b47b-451b-a680-9f9560a12481",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# Relationship\n",
        "vec = model.wv['Germany'] - model.wv['Berlin'] + model.wv['Paris']\n",
        "model.wv.most_similar([vec])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Belgium', 0.837678074836731),\n",
              " ('Paris', 0.8260020017623901),\n",
              " ('Sweden', 0.8193253874778748),\n",
              " ('France', 0.8188864588737488),\n",
              " ('Germany', 0.8029927015304565),\n",
              " ('Britain', 0.7788846492767334),\n",
              " ('Brussels', 0.7710432410240173),\n",
              " ('UK', 0.7677194476127625),\n",
              " ('Austria', 0.7147700190544128),\n",
              " ('Norway', 0.6940752267837524)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jo9c_77Wjq1o",
        "colab_type": "code",
        "outputId": "3bd66bc5-0302-4bbf-972e-7dc7c4a75ae0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# Relationship\n",
        "\n",
        "vec = model.wv['Messi'] - model.wv['Football'] + model.wv['Cricket']\n",
        "model.wv.most_similar([vec])"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Anti-poaching', 0.7540308833122253),\n",
              " ('Ratzilla', 0.7435826659202576),\n",
              " ('leapt', 0.7400335669517517),\n",
              " ('hospitalised', 0.7399585247039795),\n",
              " ('ruptures', 0.7347268462181091),\n",
              " ('Cemortan', 0.7322049140930176),\n",
              " ('Marussia', 0.7290014028549194),\n",
              " ('soggy', 0.7289555072784424),\n",
              " ('Neel', 0.726038932800293),\n",
              " ('75-year-old', 0.7176862359046936)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    }
  ]
}