Final_Working_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Final Working Model",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEfSbAA4QHas",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ec84403-f667-4a71-e737-97b041cccdee"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYsV4H8fCpZ-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e41f057c-1c38-4e9c-cd09-38db12501c0e"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0NmMdkZO8R6q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45cb5026-502b-4dfa-b5db-c0dd12ccb74c"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/91/61d69d58a1af1bd81d9ca9d62c90a6de3ab80d77f27c5df65d9a2c1f5626/transformers-4.5.0-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.2MB 13.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 45.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 55.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=0c0dbbf2a8b494224c3803519e9ad24543236847a59d7d56ee63ff7a8107a170\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKQKUWrZsjS8"
      },
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tqdm\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZNVW6xd0T0X"
      },
      "source": [
        "We'll use the `wget` package to download the dataset to the Colab instance's file system. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5m6AnuFv0QXQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ea571bd7-8893-4ffe-879a-a9af5050e704"
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp37-none-any.whl size=9681 sha256=2ac51537539af01d3205d542529f89129d458778c26aa4fdb210bea6fcd263dd\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08pO03Ff1BjI"
      },
      "source": [
        "The dataset is hosted on GitHub in this repo: https://nyu-mll.github.io/CoLA/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMtmPMkBzrvs"
      },
      "source": [
        "url = 'https://raw.githubusercontent.com/Konigari/Mixed-Initiative/master/Switchboard-Corpus/Annotation%20Assignment/Generated%20CSV/'\n",
        "lists = ['3311.csv', '4051.csv', '3367.csv', '2024.csv', '3369.csv', '3331.csv', '3584.csv', '3065.csv', '2041.csv', '3028.csv', '2040.csv', '2008.csv', '3563.csv', '4079.csv', '3382.csv', '2604.csv', '3364.csv', '3054.csv', '3325.csv', '3518.csv', '3041.csv', '2602.csv', '3344.csv', '3012.csv', '3514.csv', '4050.csv', '2631.csv', '4080.csv', '4019.csv', '2072.csv', '2586.csv', '2621.csv', '2667.csv', '2619.csv', '3570.csv', '3500.csv', '4048.csv', '2645.csv', '2039.csv', '4037.csv', '3057.csv', '2652.csv', '3338.csv', '3340.csv', '3061.csv', '2672.csv', '2647.csv', '3550.csv', '2078.csv', '3908.csv', '3508.csv', '3979.csv', '2603.csv', '2120.csv', '2038.csv', '3365.csv', '3009.csv', '3036.csv', '3064.csv', '3327.csv', '3359.csv', '3018.csv', '3040.csv', '3050.csv', '2663.csv', '3313.csv', '2648.csv', '3310.csv', '3993.csv', '4056.csv', '3595.csv', '3317.csv', '3379.csv', '4033.csv']\n",
        "urls = [url+name for name in lists]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Yv-tNv20dnH"
      },
      "source": [
        "textList = []\n",
        "labels = []\n",
        "for u in urls:\n",
        "  df = pd.read_csv(u)\n",
        "  j = 0\n",
        "  while j < (len(df['Class'])):\n",
        "    text = df['Conversation'][j]\n",
        "    i = j+1\n",
        "    try:\n",
        "\n",
        "      while df['Class'][j] == df['Class'][i]:\n",
        "        text += df['Conversation'][i] + ' '\n",
        "        i += 1\n",
        "      labels.append(df['Class'][j])\n",
        "      j = i\n",
        "      textList.append(text)\n",
        "    except:\n",
        "      break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_UkeC7SG2krJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03cc8eef-fe07-4910-8713-4e915f612077"
      },
      "source": [
        "lists = []\n",
        "fin_l = []\n",
        "sents = []\n",
        "for index,cluster in tqdm.tqdm(enumerate(textList)):\n",
        "  try:\n",
        "    if len(tokenizer.tokenize(cluster)) <= 511:\n",
        "      sents.append(cluster)\n",
        "      if labels[index] == 0:\n",
        "        fin_l.append(labels[index])\n",
        "      else:\n",
        "        fin_l.append(1)\n",
        "  except:\n",
        "    pass\n",
        "  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "698it [00:01, 417.75it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W_jkUcIasn_p"
      },
      "source": [
        "# sentences = textList\n",
        "# lb = []\n",
        "# for val in labels:\n",
        "#   if val == 0:\n",
        "#     lb.append(val)\n",
        "#   else:\n",
        "#     lb.append(1)\n",
        "# labels = lb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TIoOTMOSszG2"
      },
      "source": [
        "# Get the lists of sentences and their labels.\n",
        "sentences = sents[:632]\n",
        "labels = fin_l[:660]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "id": "kRQqTUG6twDa",
        "outputId": "2d9cb54c-c7c8-42b9-a58d-67f8b520ced7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-51-2a3a7522e540>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'shape'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z474sSC6oe7A",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7d92163d-5f64-4313-a0e5-2d7e1c92a283"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dFzmtleW6KmJ"
      },
      "source": [
        "Let's apply the tokenizer to one sentence just to see the output.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLIbudgfh6F0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e22eb12-fe16-4b89-e594-3ba640f16032"
      },
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', sentences[0])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[0]))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Original:  Our friends won't buy this analysis, let alone the next one we propose.\n",
            "Tokenized:  ['our', 'friends', 'won', \"'\", 't', 'buy', 'this', 'analysis', ',', 'let', 'alone', 'the', 'next', 'one', 'we', 'propose', '.']\n",
            "Token IDs:  [2256, 2814, 2180, 1005, 1056, 4965, 2023, 4106, 1010, 2292, 2894, 1996, 2279, 2028, 2057, 16599, 1012]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cKsH2sU0OCQA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "55f9c1fe-aecf-475b-879a-49f08ae98d7d"
      },
      "source": [
        "max_len = 0\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "\n",
        "    # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
        "    input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "\n",
        "    # Update the maximum sentence length.\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (513 > 512). Running this sequence through the model will result in indexing errors\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  513\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bBdb3pt8LuQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cfbbedd3-ff77-4995-c6d7-0282cc6f7a36"
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    \n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(labels)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Original:  All right, uh, well,let's see, How old are you Lisa? I'm twenty. Okay. How old are you? I'm older. Older Older than you are. Okay. Okay, we are supposed to talk about places we like to go. So I'm going, and where are you from, where are you calling from? I'm calling from, uh, Provo, Utah, but I'm from Plano, Texas. Oh, you are from Plano. My sister lives in Plano. Oh, really? Yes, her husband is the new Director of Admissions at, uh, University of Texas at Dallas. Oh, wow. Yeah, so, \n",
            "Token IDs: tensor([ 101, 2035, 2157, 1010, 7910, 1010, 2092, 1010, 2292, 1005, 1055, 2156,\n",
            "        1010, 2129, 2214, 2024, 2017, 7059, 1029, 1045, 1005, 1049, 3174, 1012,\n",
            "        3100, 1012, 2129, 2214, 2024, 2017, 1029, 1045, 1005, 1049, 3080, 1012,\n",
            "        3080, 3080, 2084, 2017, 2024, 1012, 3100, 1012, 3100, 1010, 2057, 2024,\n",
            "        4011, 2000, 2831, 2055, 3182, 2057, 2066, 2000, 2175, 1012, 2061, 1045,\n",
            "        1005, 1049, 2183,  102])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "d6nTQezUtEtK",
        "outputId": "7eb04088-c759-4c73-d75f-ab4a30ca4a1a"
      },
      "source": [
        "sentences"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'What all did you get for Christmas?'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRp4O7D295d_"
      },
      "source": [
        "## 3.4. Training & Validation Split\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qu0ao7p8rb06"
      },
      "source": [
        "Divide up our training set to use 90% for training and 10% for validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEgLpFVlo1Z-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53caa006-4fcf-425c-9156-33c50361b935"
      },
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "# Create a 90-10 train-validation split.\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "# Divide the dataset by randomly selecting samples.\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  568 training samples\n",
            "   64 validation samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QSeumlX4tmY-",
        "outputId": "cba8cb1b-cdf5-431f-f974-75043cd10813"
      },
      "source": [
        "input_ids.shape, attention_masks.shape, labels.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([660, 64]), torch.Size([660, 64]), torch.Size([632]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dD9i6Z2pG-sN"
      },
      "source": [
        "We'll also create an iterator for our dataset using the torch DataLoader class. This helps save on memory during training because, unlike a for loop, with an iterator the entire dataset does not need to be loaded into memory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGUqOCtgqGhP"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch \n",
        "# size of 16 or 32.\n",
        "batch_size = 32\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order. \n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
        "            batch_size = batch_size # Evaluate with this batch size.\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFsCTp_mporB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9181cd5d-5241-47e2-b079-8f57db688e0b"
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PIiVlDYCtSq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a171f4a4-5c47-401d-f5fe-41bb36b6b453"
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLs72DuMODJO"
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-p0upAhhRiIx"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4. \n",
        "# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
        "# training data.\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs]. \n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqfmWwUR_Sox"
      },
      "source": [
        "## 4.3. Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cQNvaZ9bnyy"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpt6tR83keZD"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6J-FYdx6nFE_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eeb08a8f-aded-4ec8-a511-f23a5786c3cf"
      },
      "source": [
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss, \n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # In PyTorch, calling `model` will in turn call the model's `forward` \n",
        "        # function and pass down the arguments. The `forward` function is \n",
        "        # documented here: \n",
        "        # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification\n",
        "        # The results are returned in a results object, documented here:\n",
        "        # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput\n",
        "        # Specifically, we'll get the loss (because we provided labels) and the\n",
        "        # \"logits\"--the model outputs prior to activation.\n",
        "        result = model(b_input_ids, \n",
        "                       token_type_ids=None, \n",
        "                       attention_mask=b_input_mask, \n",
        "                       labels=b_labels,\n",
        "                       return_dict=True)\n",
        "\n",
        "        loss = result.loss\n",
        "        logits = result.logits\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            result = model(b_input_ids, \n",
        "                           token_type_ids=None, \n",
        "                           attention_mask=b_input_mask,\n",
        "                           labels=b_labels,\n",
        "                           return_dict=True)\n",
        "\n",
        "        # Get the loss and \"logits\" output by the model. The \"logits\" are the \n",
        "        # output values prior to applying an activation function like the \n",
        "        # softmax.\n",
        "        loss = result.loss\n",
        "        logits = result.logits\n",
        "            \n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "        \n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    \n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.84\n",
            "  Training epcoh took: 0:00:04\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 0.82\n",
            "  Validation took: 0:00:00\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.84\n",
            "  Training epcoh took: 0:00:04\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 0.82\n",
            "  Validation took: 0:00:00\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.84\n",
            "  Training epcoh took: 0:00:04\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 0.82\n",
            "  Validation took: 0:00:00\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.84\n",
            "  Training epcoh took: 0:00:04\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 0.82\n",
            "  Validation took: 0:00:00\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:00:15 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAN0LZBOOPVh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "208fe3ee-e6ca-46d9-835b-e7e4955a8d6d"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "# Create sentence and label lists\n",
        "sentences = [\"I, I, You see, I'm from west Texas. but bait tends to work the best, just some blood bait. Oh, are you? Yeah. Where are you from? Lubbock. Oh, I'm from Midland. Oh, another west Texan. I went to college at Tech,\"]\n",
        "\n",
        "labels = [1]\n",
        "\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    \n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(labels)\n",
        "\n",
        "# Set the batch size.  \n",
        "batch_size = 32  \n",
        "\n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "16lctEOyNFik"
      },
      "source": [
        "## 5.2. Evaluate on Test Set\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hba10sXR7Xi6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4d7cf0a-1b2e-47f4-de46-da06212e7052"
      },
      "source": [
        "# Prediction on test set\n",
        "\n",
        "print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))\n",
        "\n",
        "# Put model in evaluation mode\n",
        "model.eval()\n",
        "\n",
        "# Tracking variables \n",
        "predictions , true_labels = [], []\n",
        "\n",
        "# Predict \n",
        "for batch in prediction_dataloader:\n",
        "  # Add batch to GPU\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  # Telling the model not to compute or store gradients, saving memory and \n",
        "  # speeding up prediction\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions.\n",
        "      result = model(b_input_ids, \n",
        "                     token_type_ids=None, \n",
        "                     attention_mask=b_input_mask,\n",
        "                     return_dict=True)\n",
        "\n",
        "  logits = result.logits\n",
        "\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting labels for 1 test sentences...\n",
            "    DONE.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kKazUBbRvgpk",
        "outputId": "d3a216ef-d48c-4e7f-95f8-5c6a05eca3d9"
      },
      "source": [
        "predictions"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([[ 0.7452995 , -0.14377806]], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q2079Qyn8Mt8"
      },
      "source": [
        "## A1. Saving & Loading Fine-Tuned Model\n",
        "\n",
        "This first cell (taken from `run_glue.py` [here](https://github.com/huggingface/transformers/blob/35ff345fc9df9e777b27903f11fa213e4052595b/examples/run_glue.py#L495)) writes the model and tokenizer out to disk."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ulTWaOr8QNY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b73b37b-2598-4992-d6d7-0649f410b5c0"
      },
      "source": [
        "import os\n",
        "\n",
        "# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
        "\n",
        "output_dir = './model_save/'\n",
        "\n",
        "# Create output directory if needed\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
        "# They can then be reloaded using `from_pretrained()`\n",
        "model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)\n",
        "\n",
        "# Good practice: save your training arguments together with the trained model\n",
        "# torch.save(args, os.path.join(output_dir, 'training_args.bin'))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving model to ./model_save/\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./model_save/tokenizer_config.json',\n",
              " './model_save/special_tokens_map.json',\n",
              " './model_save/vocab.txt',\n",
              " './model_save/added_tokens.json')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-tjHkR7lc1I"
      },
      "source": [
        "Let's check out the file sizes, out of curiosity."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqMzI3VTCZo5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96104fe5-67d0-4310-d778-58da5194c2e1"
      },
      "source": [
        "!ls -l --block-size=K ./model_save/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 428000K\n",
            "-rw-r--r-- 1 root root      1K Feb  2 17:10 config.json\n",
            "-rw-r--r-- 1 root root 427757K Feb  2 17:10 pytorch_model.bin\n",
            "-rw-r--r-- 1 root root      1K Feb  2 17:10 special_tokens_map.json\n",
            "-rw-r--r-- 1 root root      1K Feb  2 17:10 tokenizer_config.json\n",
            "-rw-r--r-- 1 root root    227K Feb  2 17:10 vocab.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fr_bt2rFlgDn"
      },
      "source": [
        "The largest file is the model weights, at around 418 megabytes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-WUFUIQ8Cu8D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0c9b6c6-5fb8-4d61-d28a-be4324be5a5b"
      },
      "source": [
        "!ls -l --block-size=M ./model_save/pytorch_model.bin"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 418M Feb  2 17:10 ./model_save/pytorch_model.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dzGKvOFAll_e"
      },
      "source": [
        "To save your model across Colab Notebook sessions, download it to your local machine, or ideally copy it to your Google Drive."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Trr-A-POC18_"
      },
      "source": [
        "# Mount Google Drive to this Notebook instance.\n",
        "from google.colab import drive\n",
        "    drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NxlZsafTC-V5"
      },
      "source": [
        "# Copy the model files to a directory in your Google Drive.\n",
        "!cp -r ./model_save/ \"./drive/Shared drives/ChrisMcCormick.AI/Blog Posts/BERT Fine-Tuning/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W0vstijw85SZ"
      },
      "source": [
        "The following functions will load the model back from disk."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nskPzUM084zL"
      },
      "source": [
        "# Load a trained model and vocabulary that you have fine-tuned\n",
        "model = model_class.from_pretrained(output_dir)\n",
        "tokenizer = tokenizer_class.from_pretrained(output_dir)\n",
        "\n",
        "# Copy the model to the GPU.\n",
        "model.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NIWouvDrGVAi"
      },
      "source": [
        "## A.2. Weight Decay\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f123ZAlF1OyW"
      },
      "source": [
        "The huggingface example includes the following code block for enabling weight decay, but the default decay rate is \"0.0\", so I moved this to the appendix.\n",
        "\n",
        "This block essentially tells the optimizer to not apply weight decay to the bias terms (e.g., $ b $ in the equation $ y = Wx + b $ ). Weight decay is a form of regularization--after calculating the gradients, we multiply them by, e.g., 0.99."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxSMw0FrptiL"
      },
      "source": [
        "# This code is taken from:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L102\n",
        "\n",
        "# Don't apply weight decay to any parameters whose names include these tokens.\n",
        "# (Here, the BERT doesn't have `gamma` or `beta` parameters, only `bias` terms)\n",
        "no_decay = ['bias', 'LayerNorm.weight']\n",
        "\n",
        "# Separate the `weight` parameters from the `bias` parameters. \n",
        "# - For the `weight` parameters, this specifies a 'weight_decay_rate' of 0.01. \n",
        "# - For the `bias` parameters, the 'weight_decay_rate' is 0.0. \n",
        "optimizer_grouped_parameters = [\n",
        "    # Filter for all parameters which *don't* include 'bias', 'gamma', 'beta'.\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.1},\n",
        "    \n",
        "    # Filter for parameters which *do* include those.\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]\n",
        "\n",
        "# Note - `optimizer_grouped_parameters` only includes the parameter values, not \n",
        "# the names."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKzLS9ohzGVu"
      },
      "source": [
        "# Revision History"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SZqpiHEnGqYR"
      },
      "source": [
        "**Version 4** - *Feb 2nd, 2020* - (current)\n",
        "* Updated all calls to `model` (fine-tuning and evaluation) to use the [`SequenceClassifierOutput`](https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput) class.\n",
        "* Moved illustration images to Google Drive--Colab appears to no longer support images at external URLs.\n",
        "\n",
        "**Version 3** - *Mar 18th, 2020*\n",
        "* Simplified the tokenization and input formatting (for both training and test) by leveraging the `tokenizer.encode_plus` function. \n",
        "`encode_plus` handles padding *and* creates the attention masks for us.\n",
        "* Improved explanation of attention masks.\n",
        "* Switched to using `torch.utils.data.random_split` for creating the training-validation split.\n",
        "* Added a summary table of the training statistics (validation loss, time per epoch, etc.).\n",
        "* Added validation loss to the learning curve plot, so we can see if we're overfitting. \n",
        "    * Thank you to [Stas Bekman](https://ca.linkedin.com/in/stasbekman) for contributing this!\n",
        "* Displayed the per-batch MCC as a bar plot.\n",
        "\n",
        "**Version 2** - *Dec 20th, 2019* - [link](https://colab.research.google.com/drive/1Y4o3jh3ZH70tl6mCd76vz_IxX23biCPP)\n",
        "* huggingface renamed their library to `transformers`. \n",
        "* Updated the notebook to use the `transformers` library.\n",
        "\n",
        "**Version 1** - *July 22nd, 2019*\n",
        "* Initial version."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FL_NnDGxRpEI"
      },
      "source": [
        "## Further Work\n",
        "\n",
        "* It might make more sense to use the MCC score for “validation accuracy”, but I’ve left it out so as not to have to explain it earlier in the Notebook.\n",
        "* Seeding -- I’m not convinced that setting the seed values at the beginning of the training loop is actually creating reproducible results…\n",
        "* The MCC score seems to vary substantially across different runs. It would be interesting to run this example a number of times and show the variance.\n"
      ]
    }
  ]
}