bert_tf_imdb_baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bert-tf-imdb-baseline.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNRnaA8yhRsjzjB1tRUL22B",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anwesham-lab/cs-229-230-project/blob/main/bert_tf_imdb_baseline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1MIw4H9OGfq4",
        "outputId": "0eab21da-d3ea-422a-a396-4c2cf376d985"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting bert-for-tf2\n",
            "  Downloading bert-for-tf2-0.14.9.tar.gz (41 kB)\n",
            "\u001b[?25l\r\u001b[K     |████████                        | 10 kB 32.5 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 20 kB 20.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 30 kB 16.7 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 40 kB 15.0 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 41 kB 180 kB/s \n",
            "\u001b[?25hCollecting py-params>=0.9.6\n",
            "  Downloading py-params-0.10.2.tar.gz (7.4 kB)\n",
            "Collecting params-flow>=0.8.0\n",
            "  Downloading params-flow-0.8.2.tar.gz (22 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (1.21.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (4.64.0)\n",
            "Building wheels for collected packages: bert-for-tf2, params-flow, py-params\n",
            "  Building wheel for bert-for-tf2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for bert-for-tf2: filename=bert_for_tf2-0.14.9-py3-none-any.whl size=30535 sha256=546909053ee7492ca7fcddfb58e5ed205a5a4a0a682c7822de8a728a1f430797\n",
            "  Stored in directory: /root/.cache/pip/wheels/47/b6/e5/8c76ec779f54bc5c2f1b57d2200bb9c77616da83873e8acb53\n",
            "  Building wheel for params-flow (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for params-flow: filename=params_flow-0.8.2-py3-none-any.whl size=19472 sha256=d3818a205450826937fe2d8189c2a08b046faef7a37c4b416ae36b2446001929\n",
            "  Stored in directory: /root/.cache/pip/wheels/0e/fc/d2/a44fff33af0f233d7def6e7de413006d57c10e10ad736fe8f5\n",
            "  Building wheel for py-params (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-params: filename=py_params-0.10.2-py3-none-any.whl size=7911 sha256=3d110726dc3438b821108bddd775f9e19a95005c55be4f7c35392da6c7f08b6c\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/11/67/33cc51bbee127cb8fb2ba549cd29109b2f22da43ddf9969716\n",
            "Successfully built bert-for-tf2 params-flow py-params\n",
            "Installing collected packages: py-params, params-flow, bert-for-tf2\n",
            "Successfully installed bert-for-tf2-0.14.9 params-flow-0.8.2 py-params-0.10.2\n"
          ]
        }
      ],
      "source": [
        "!pip install bert-for-tf2"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rF-qMpnpGoAv",
        "outputId": "3399d7b5-f140-40d5-e541-9f023c61f676"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/My\\ Drive/230\n",
        "%pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "JqhzROEWGrSS",
        "outputId": "45562928-2a93-4701-ed47-444d6e344101"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/My Drive/230\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/drive/My Drive/230'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import modules\n",
        "import os\n",
        "import bert\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.models import  Model\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard\n",
        "from tqdm import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "print(\"TensorFlow Version:\",tf.__version__)\n",
        "print(\"Hub version: \",hub.__version__)\n",
        "pd.set_option('display.max_colwidth',1000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9hd5uQZ2HVQ9",
        "outputId": "b916f1ac-55a8-4b01-b2e9-4cfe72ce7f77"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TensorFlow Version: 2.8.0\n",
            "Hub version:  0.12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('imdb_data_shuffled.csv', names=['sentiment', 'review'])"
      ],
      "metadata": {
        "id": "KYCIiaEuHnsW"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6cTDggsRHzJx",
        "outputId": "82758f11-f528-4c8c-8629-ebf9fb9e2a5c"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['sentiment', 'review'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"sentiment\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94Ws1ATlH9K5",
        "outputId": "f1d7992c-d4f8-4ae4-a5fb-88b7068e2a58"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "positive    25000\n",
              "negative    25000\n",
              "Name: sentiment, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MAX_SEQ_LEN = 0\n",
        "seq_len = []\n",
        "for sent in df[\"review\"]:\n",
        "    if len(sent) > MAX_SEQ_LEN: MAX_SEQ_LEN = len(sent)\n",
        "    seq_len.append(len(sent))\n",
        "print(MAX_SEQ_LEN)\n",
        "print(sum(seq_len)/len(df))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4IbR013UIEGx",
        "outputId": "494ec394-a78f-47e0-f9aa-05c70dbe570b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13704\n",
            "1309.43102\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# https://www.tensorflow.org/official_models/fine_tuning_bert\n",
        "gs_folder_bert = \"gs://cloud-tpu-checkpoints/bert/v3/uncased_L-12_H-768_A-12\"\n",
        "tf.io.gfile.listdir(gs_folder_bert)\n",
        "tokenizer = bert.bert_tokenization.FullTokenizer(vocab_file = os.path.join(gs_folder_bert, \"vocab.txt\"), do_lower_case=True)\n",
        "print(\"Vocab size:\", len(tokenizer.vocab))\n",
        "tokens = tokenizer.tokenize(\"Hello TensorFlow!\")\n",
        "print(tokens)\n",
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(ids)\n",
        "print(tokenizer.convert_tokens_to_ids(['[CLS]', '[SEP]']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M7hdaHk-IKzs",
        "outputId": "e2fda08e-8d72-4133-975c-9ef406f42405"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vocab size: 30522\n",
            "['hello', 'tensor', '##flow', '!']\n",
            "[7592, 23435, 12314, 999]\n",
            "[101, 102]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_tonkenizer(bert_layer):\n",
        "    \"\"\"Instantiate Tokenizer with vocab\"\"\"\n",
        "    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "    do_lower_case = bert_layer.resolved_object.do_lower_case.numpy() \n",
        "    tokenizer = bert.bert_tokenization.FullTokenizer(vocab_file, do_lower_case)\n",
        "    print(\"Vocab size:\", len(tokenizer.vocab))\n",
        "    return tokenizer"
      ],
      "metadata": {
        "id": "muEypBU-IOAx"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_ids(tokens, tokenizer, MAX_SEQ_LEN):\n",
        "    \"\"\"Token ids from Tokenizer vocab\"\"\"\n",
        "    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "    input_ids = token_ids + [0] * (MAX_SEQ_LEN - len(token_ids))\n",
        "    return input_ids"
      ],
      "metadata": {
        "id": "IgDrhbTsIROn"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_masks(tokens, MAX_SEQ_LEN):\n",
        "    \"\"\"Masks: 1 for real tokens and 0 for paddings\"\"\"\n",
        "    return [1] * len(tokens) + [0] * (MAX_SEQ_LEN - len(tokens))\n",
        "\n",
        "print(get_masks(tokenizer.tokenize(\"Hello TensorFlow!\"), 500))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ckOxVRhKMvr0",
        "outputId": "1e944882-2eb4-4c5e-c7e2-28d8bfb9fc49"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_segments(tokens, MAX_SEQ_LEN):\n",
        "    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"  \n",
        "    segments = []\n",
        "    current_segment_id = 0\n",
        "    for token in tokens:\n",
        "        segments.append(current_segment_id)\n",
        "        if token == \"[SEP]\":\n",
        "            current_segment_id = 1\n",
        "    return segments + [0] * (MAX_SEQ_LEN - len(tokens))\n"
      ],
      "metadata": {
        "id": "u9MLujQbM3R7"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_single_input(sentence, tokenizer, max_len):\n",
        "    \"\"\"Create an input from a sentence\"\"\"\n",
        "    stokens = tokenizer.tokenize(sentence)\n",
        "    stokens = stokens[:max_len] # max_len = MAX_SEQ_LEN - 2, why -2 ? ans: reserved for [CLS] & [SEP]\n",
        "    stokens = [\"[CLS]\"] + stokens + [\"[SEP]\"]\n",
        "    return get_ids(stokens, tokenizer, max_len+2), get_masks(stokens, max_len+2), get_segments(stokens, max_len+2)"
      ],
      "metadata": {
        "id": "Y2mTt6W-M8_B"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_sentences_to_features(sentences, tokenizer, MAX_SEQ_LEN):\n",
        "    \"\"\"Convert sentences to features: input_ids, input_masks and input_segments\"\"\"\n",
        "    input_ids, input_masks, input_segments = [], [], []\n",
        "    for sentence in tqdm(sentences, position=0, leave=True):\n",
        "      ids, masks, segments = create_single_input(sentence, tokenizer, MAX_SEQ_LEN-2) # why -2 ? ans: reserved for [CLS] & [SEP]\n",
        "      assert len(ids) == MAX_SEQ_LEN\n",
        "      assert len(masks) == MAX_SEQ_LEN\n",
        "      assert len(segments) == MAX_SEQ_LEN\n",
        "      input_ids.append(ids)\n",
        "      input_masks.append(masks)\n",
        "      input_segments.append(segments)\n",
        "    return [np.asarray(input_ids, dtype=np.int32), np.asarray(input_masks, dtype=np.int32), np.asarray(input_segments, dtype=np.int32)]\n",
        "\n",
        "# checkfunality of convert_sentences_to_features\n",
        "[res1, res2, res3] = convert_sentences_to_features([\"Hello TensorFlow!\"], tokenizer, 500)\n",
        "print(res1, res2, res3)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iIuimfXwNApZ",
        "outputId": "b4fcd946-7828-4980-b113-e15f08e58a63"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00, 3350.08it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[  101  7592 23435 12314   999   102     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0     0     0     0     0\n",
            "      0     0     0     0     0     0     0     0]] [[1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]] [[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model"
      ],
      "metadata": {
        "id": "CuLGGDiPNH2M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def bert_tf_model(bert_base):\n",
        "    # Load the pre-trained BERT base model\n",
        "    bert_layer = hub.KerasLayer(handle=bert_base, trainable=True)  \n",
        "    # BERT layer three inputs: ids, masks and segments\n",
        "    input_ids = Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name=\"input_ids\")           \n",
        "    input_masks = Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name=\"input_masks\")       \n",
        "    input_segments = Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name=\"segment_ids\")\n",
        "\n",
        "    inputs = [input_ids, input_masks, input_segments] # BERT inputs\n",
        "    pooled_output, sequence_output = bert_layer(inputs) # BERT outputs\n",
        "    \n",
        "    x = Dense(units=768, activation='relu')(pooled_output) # hidden layer \n",
        "    x = Dropout(0.15)(x) \n",
        "    outputs = Dense(2, activation=\"softmax\")(x) # output layer\n",
        "\n",
        "    model = Model(inputs=inputs, outputs=outputs)\n",
        "    return model"
      ],
      "metadata": {
        "id": "K8JUgE9FNKPC"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# create tokenizer\n",
        "gs_folder_bert = \"gs://cloud-tpu-checkpoints/bert/v3/uncased_L-12_H-768_A-12\"\n",
        "tf.io.gfile.listdir(gs_folder_bert)\n",
        "tokenizer = bert.bert_tokenization.FullTokenizer(vocab_file = os.path.join(gs_folder_bert, \"vocab.txt\"), do_lower_case=True)\n",
        "\n",
        "# create training data and testing data\n",
        "df = df.sample(frac=1) # Shuffle the dataset\n",
        "x_train = convert_sentences_to_features(df['review'][:40000], tokenizer, MAX_SEQ_LEN)\n",
        "x_valid = convert_sentences_to_features(df['review'][40000:45000], tokenizer, MAX_SEQ_LEN)\n",
        "x_test = convert_sentences_to_features(df['review'][45000:], tokenizer, MAX_SEQ_LEN)\n",
        "df['sentiment'].replace('positive', 1., inplace=True)\n",
        "df['sentiment'].replace('negative', 0., inplace=True)\n",
        "one_hot_encoded = to_categorical(df['sentiment'].values)\n",
        "y_train = one_hot_encoded[:40000]\n",
        "y_valid = one_hot_encoded[40000:45000]\n",
        "y_test =  one_hot_encoded[45000:]\n",
        "\n",
        "# hyper-parameters\n",
        "BATCH_SIZE = 8\n",
        "EPOCHS = 1\n",
        "MAX_SEQ_LEN = 500\n",
        "\n",
        "# model construction\n",
        "bert_base = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\"\n",
        "model = bert_tf_model(bert_base) \n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EgFtrzzbNN3R",
        "outputId": "ef3d9b4f-506e-47f4-8e33-22668da030bd"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 40000/40000 [03:05<00:00, 215.93it/s]\n",
            "100%|██████████| 5000/5000 [00:20<00:00, 239.69it/s]\n",
            "100%|██████████| 5000/5000 [00:20<00:00, 241.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_ids (InputLayer)         [(None, 500)]        0           []                               \n",
            "                                                                                                  \n",
            " input_masks (InputLayer)       [(None, 500)]        0           []                               \n",
            "                                                                                                  \n",
            " segment_ids (InputLayer)       [(None, 500)]        0           []                               \n",
            "                                                                                                  \n",
            " keras_layer_2 (KerasLayer)     [(None, 768),        109482241   ['input_ids[0][0]',              \n",
            "                                 (None, 500, 768)]                'input_masks[0][0]',            \n",
            "                                                                  'segment_ids[0][0]']            \n",
            "                                                                                                  \n",
            " dense_4 (Dense)                (None, 768)          590592      ['keras_layer_2[0][0]']          \n",
            "                                                                                                  \n",
            " dropout_2 (Dropout)            (None, 768)          0           ['dense_4[0][0]']                \n",
            "                                                                                                  \n",
            " dense_5 (Dense)                (None, 2)            1538        ['dropout_2[0][0]']              \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 110,074,371\n",
            "Trainable params: 110,074,370\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# use adam optimizer to minimize the categorical_crossentropy loss\n",
        "optimizer = Adam(learning_rate=2e-5)\n",
        "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "# fit the data to the model\n",
        "history = model.fit(x_train, y_train, validation_data=(x_valid, y_valid), epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=1)\n",
        "\n",
        "# save the trained model\n",
        "model.save('imdb_bert_fine_tune.h5')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mJvLoWAfNRj7",
        "outputId": "bd600e0f-6c62-4e29-c2b9-9e36a1d40d7f"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5000/5000 [==============================] - 4855s 966ms/step - loss: 0.2056 - accuracy: 0.9191 - val_loss: 0.1856 - val_accuracy: 0.9322\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load the pretrained nlp_model\n",
        "from tensorflow.keras.models import load_model\n",
        "imdb_model = load_model('imdb_bert_fine_tune.h5', custom_objects={'KerasLayer': hub.KerasLayer})\n",
        "\n",
        "# predict on test dataset\n",
        "from sklearn.metrics import classification_report\n",
        "y_pred = np.argmax(imdb_model.predict(x_test), axis=1)"
      ],
      "metadata": {
        "id": "aVhHPq_Qp13S"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(classification_report(np.argmax(y_test, axis=1), y_pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9OPZx1uvp9XZ",
        "outputId": "022c3483-6366-41ae-95fe-e226b0468dba"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.91      0.93      2496\n",
            "           1       0.91      0.96      0.94      2504\n",
            "\n",
            "    accuracy                           0.93      5000\n",
            "   macro avg       0.94      0.93      0.93      5000\n",
            "weighted avg       0.94      0.93      0.93      5000\n",
            "\n"
          ]
        }
      ]
    }
  ]
}