Bilal_et_al_baseline_on_Yelp_data.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YXAL6gpkijkz",
        "outputId": "df92484c-9e1b-4c55-b4fc-12545655716d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 24.2 MB 683 kB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 12.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 13.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 101 kB 10.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 60.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 62.5 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install gensim==3.8.3 --quiet\n",
        "!pip install pydot --quiet\n",
        "!pip install sentencepiece --quiet\n",
        "!pip install transformers --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "Kd0xo5RHVbg-"
      },
      "outputs": [],
      "source": [
        "from google.cloud import storage\n",
        "import google.oauth2.credentials\n",
        "import json\n",
        "import seaborn as sns\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.layers import Embedding, Input, Dense, Lambda\n",
        "from tensorflow.keras.models import Model\n",
        "import tensorflow.keras.backend as K\n",
        "import tensorflow_datasets as tfds\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "import torch\n",
        "\n",
        "import sklearn as sk\n",
        "import nltk\n",
        "from nltk.corpus import reuters\n",
        "from nltk.data import find\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import re\n",
        "\n",
        "#This continues to work with gensim 3.8.3.  It doesn't yet work with 4.x.  \n",
        "#Make sure your pip install command specifies gensim==3.8.3\n",
        "import gensim\n",
        "\n",
        "from transformers import BertTokenizer, TFBertModel, XLNetTokenizer, TFXLNetForSequenceClassification, TFBertForSequenceClassification, TFRobertaForSequenceClassification, RobertaTokenizer\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ouPkZabHYvwR"
      },
      "source": [
        "## Data loading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Umy49AUj-QH",
        "outputId": "3541b9a4-ff6a-466b-c9dc-db27e80b47dd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Added this so I can just put the data files in my google drive and access them from there, if you have a preferred way of storing the data feel free to change\n",
        "# You can download the data here: \n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "I-Eth0DBZQGj"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "train_bilal = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/Paper_Data/train.csv\", encoding='latin-1')\n",
        "test_bilal = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/Paper_Data/test.csv\", encoding='latin-1')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "1eYqsg-aXkJi"
      },
      "outputs": [],
      "source": [
        "x_train_bilal = list(train_bilal[['sentence']].sentence)\n",
        "y_train_bilal = np.asarray(train_bilal[['label']].label)\n",
        "\n",
        "x_test_bilal = list(test_bilal[['sentence']].sentence)\n",
        "y_test_bilal = np.asarray(test_bilal[['label']].label)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "wbc7hIRSCu8z"
      },
      "outputs": [],
      "source": [
        "x_train_bilal, x_valid_bilal, y_train_bilal, y_valid_bilal = train_test_split(x_train_bilal, y_train_bilal, train_size=0.9)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q_CeqQK99h7j"
      },
      "source": [
        "## BERT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236,
          "referenced_widgets": [
            "80aaf35d190840bf8015f3b63a35c4ba",
            "a8d14e05375d489c8f0787e536a1782c",
            "26b05e65a22340c59e17994c4c932028",
            "96977700202e4583b0fcc4d6776a69ce",
            "be58acda3e3f4712bcb72bbfc3b338e2",
            "239dd327934d4d72b2903628a66d556b",
            "e491401166184f619e0f8955f528146d",
            "49b9c3d055bb45d79245ac97655d3fe3",
            "bb070356766a42248b882448599da076",
            "d8ec0c78cfa144c5aeee550a58dc93ba",
            "6ad93e985d4f4ff1ada966ff6afa866f",
            "6505fb018b3b4a0ea1012463f82df3ca",
            "d24e921fbe514544a433889da4f421f8",
            "e109fca25373425091ce0fa33fa2d19c",
            "1611fe24ee014a4797b3559d73ab4630",
            "bfe89e218bca4b50810e45464e1d986d",
            "2b486d2bbacb4060af870dd55fb7c283",
            "e7580bf4b52c44a09e1e304e07adec11",
            "c976d7bdb29d416185262635c550a35c",
            "64244d3f83e34375b56c2ca460d04bf5",
            "68c3ec9c4d0d49ef96b08e46854e88c5",
            "23ebc1b675d14aba808674566807cfc0",
            "62c92f8225db48709bfa43f8274a68ec",
            "86d44229d4cb4debb4052cd317bcb4c4",
            "4a3079c3a155423d94bbabf91e7867c5",
            "9747aef346124bcf849cafa15718dbf7",
            "e44b238365204e4eb735f34256850aff",
            "7e208b5afaf9403eafb7ba4103b93741",
            "18fd148e30844b389970e37a2af2722b",
            "1e96b8685d494462b721deb58ab819c3",
            "bb6334152a84438d810525fa2c158e3a",
            "46cdab10ea3e4a0d8442baeea4da4e06",
            "a499917149ee4a45bfcfa71ea229a839",
            "c0c7c35d9835431b9a0811dfee57bac0",
            "239747c48e7941d98d0c39ad65c1300e",
            "e42fd8bf70e3491e80bce582e6769f20",
            "b53fbf0590c948dc86c21b3053f19d3b",
            "241d5d1d76064fe89ebe24152da6d486",
            "288a1c26e2b04ce0b1ffe839cf63cf66",
            "35d769c50be7437683162890085cc428",
            "56d4687fc2f4441fa0a977cbc87b15b6",
            "6efec14161ee40abb152c7d089cd8b8e",
            "c918b4ecebac4668b686809b0debc294",
            "aa12a5021ab747b9afdd35189ae93217"
          ]
        },
        "id": "yAU0ev0M9kMB",
        "outputId": "a6f839a4-ac1b-4324-99e6-e9f8dfb2c0d2"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "80aaf35d190840bf8015f3b63a35c4ba"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6505fb018b3b4a0ea1012463f82df3ca"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "62c92f8225db48709bfa43f8274a68ec"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/511M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c0c7c35d9835431b9a0811dfee57bac0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "model_checkpoint = 'bert-base-uncased'\n",
        "bert_uctokenizer = BertTokenizer.from_pretrained(model_checkpoint)\n",
        "bert_model = TFBertForSequenceClassification.from_pretrained(model_checkpoint)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "8WlVcT5PCKvX"
      },
      "outputs": [],
      "source": [
        "max_length = 320\n",
        "train_encodings_bilal = bert_uctokenizer(x_train_bilal, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')\n",
        "\n",
        "valid_encodings_bilal = bert_uctokenizer(x_valid_bilal, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')\n",
        "\n",
        "test_encodings_bilal = bert_uctokenizer(x_test_bilal, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Baseline model"
      ],
      "metadata": {
        "id": "x-EDHRksw930"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "WoZGoWJtCKya"
      },
      "outputs": [],
      "source": [
        "def bilal_bert_model():\n",
        "    \"\"\"Create a BERT model using the model and parameters specified in the Bilal paper:\n",
        "    https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "\n",
        "        - model: TFBertForSequenceClassification\n",
        "        - learning rate: 2e-5\n",
        "        - epsilon: 1e-8\n",
        "    \"\"\"\n",
        "    # Using the TFBertForSequenceClassification as specified in the paper:\n",
        "    bert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "\n",
        "    # Freeze all layers except the last 4 which are the pooled classification layers:\n",
        "    untrainable = [w.name for w in bert_model.weights[:-4]]\n",
        "    trainable = [w.name for w in bert_model.weights[-4:]]\n",
        "\n",
        "    for w in bert_model.weights:\n",
        "        if w.name in untrainable:\n",
        "            w._trainable = False\n",
        "        elif w.name in trainable:\n",
        "            w._trainable = True\n",
        "\n",
        "    # Compile the model:\n",
        "    bert_model.compile(\n",
        "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,epsilon=1e-08),\n",
        "        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
        "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")]\n",
        "    )\n",
        "\n",
        "    return bert_model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "VlvbesOmCYl1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f3066b7-5d73-4bf9-95e9-da15eeee5619"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "bilal_bert_model = bilal_bert_model()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "bilal_bert_model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MYrNDzDDcR6t",
        "outputId": "db777f6a-e981-4d2f-845a-069ee31a5373"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"tf_bert_for_sequence_classification_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  109482240 \n",
            "                                                                 \n",
            " dropout_75 (Dropout)        multiple                  0         \n",
            "                                                                 \n",
            " classifier (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train the model using the specifications from the paper: https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "# -- epochs = 4\n",
        "# -- batch_size = 32\n",
        "\n",
        "history = bilal_bert_model.fit(\n",
        "    [train_encodings_bilal.input_ids, train_encodings_bilal.token_type_ids, train_encodings_bilal.attention_mask], \n",
        "    y_train_bilal,\n",
        "    validation_data=(\n",
        "        [valid_encodings_bilal.input_ids, valid_encodings_bilal.token_type_ids, valid_encodings_bilal.attention_mask], \n",
        "        y_valid_bilal\n",
        "        ),\n",
        "    batch_size=32, \n",
        "    epochs=4\n",
        ")"
      ],
      "metadata": {
        "id": "wHw31flmxflU",
        "outputId": "6eb05048-fcb1-4f6a-d93e-6562255e4257",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_for_sequence_classification_1/bert/embeddings/word_embeddings/weight:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/token_type_embeddings/embeddings:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/position_embeddings/embeddings:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/LayerNorm/beta:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_for_sequence_classification_1/bert/embeddings/word_embeddings/weight:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/token_type_embeddings/embeddings:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/position_embeddings/embeddings:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/embeddings/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._0/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._1/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._2/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._3/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._4/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._5/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._6/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._7/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._8/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._9/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._10/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/query/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/key/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/self/value/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/dense/kernel:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/dense/bias:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification_1/bert/encoder/layer_._11/output/LayerNorm/beta:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "225/225 [==============================] - 123s 506ms/step - loss: 0.6246 - accuracy: 0.6549 - val_loss: 0.6044 - val_accuracy: 0.6975\n",
            "Epoch 2/4\n",
            "225/225 [==============================] - 112s 497ms/step - loss: 0.5981 - accuracy: 0.6871 - val_loss: 0.5970 - val_accuracy: 0.6913\n",
            "Epoch 3/4\n",
            "225/225 [==============================] - 112s 497ms/step - loss: 0.5960 - accuracy: 0.6906 - val_loss: 0.5939 - val_accuracy: 0.6925\n",
            "Epoch 4/4\n",
            "225/225 [==============================] - 112s 497ms/step - loss: 0.5902 - accuracy: 0.6918 - val_loss: 0.5890 - val_accuracy: 0.6975\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score = bilal_bert_model.evaluate([test_encodings_bilal.input_ids, test_encodings_bilal.token_type_ids, test_encodings_bilal.attention_mask], y_test_bilal)\n",
        "\n",
        "print(\"Test loss:\", score[0])\n",
        "print(\"Test accuracy:\", score[1])"
      ],
      "metadata": {
        "id": "uoFfaK-9xwsP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83bb73fd-5f21-461b-fafc-7f34f15f5299"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "63/63 [==============================] - 31s 433ms/step - loss: 0.5954 - accuracy: 0.6980\n",
            "Test loss: 0.5953600406646729\n",
            "Test accuracy: 0.6980000138282776\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = bilal_bert_model.predict([test_encodings_bilal.input_ids, test_encodings_bilal.token_type_ids, test_encodings_bilal.attention_mask])\n",
        "preds = predictions.to_tuple()[0].argmax(1)\n",
        "print(classification_report(y_test_bilal, preds))"
      ],
      "metadata": {
        "id": "L601qJxDx1K3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03f5d256-a1a0-4970-f884-9ff4df6bec28"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.69      0.71      0.70      1000\n",
            "           1       0.70      0.69      0.69      1000\n",
            "\n",
            "    accuracy                           0.70      2000\n",
            "   macro avg       0.70      0.70      0.70      2000\n",
            "weighted avg       0.70      0.70      0.70      2000\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Control baseline with larger Yelp data set "
      ],
      "metadata": {
        "id": "t5q-4sIbyCQV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_train.csv\")\n",
        "test = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_test.csv\")\n",
        "valid = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_valid.csv\")"
      ],
      "metadata": {
        "id": "RuHXI9hByBSX"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = list(train[['text']].text)\n",
        "y_train = np.asarray(train[['label']].label)\n",
        "\n",
        "x_test = list(test[['text']].text)\n",
        "y_test = np.asarray(test[['label']].label)\n",
        "\n",
        "x_valid = list(valid[['text']].text)\n",
        "y_valid = np.asarray(valid[['label']].label)"
      ],
      "metadata": {
        "id": "WZRn3TO4yBim"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_length = 320\n",
        "train_encodings = bert_uctokenizer(x_train, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')\n",
        "\n",
        "valid_encodings = bert_uctokenizer(x_valid, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')\n",
        "\n",
        "test_encodings = bert_uctokenizer(x_test, \n",
        "              max_length=max_length,\n",
        "              truncation=True,\n",
        "              padding='max_length', \n",
        "              return_tensors='tf')"
      ],
      "metadata": {
        "id": "yAdpmuoJ_L-x"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bilal_bert_model_expanded = bilal_bert_model()"
      ],
      "metadata": {
        "id": "2a_0vU0f_M0b",
        "outputId": "48c30b92-30c7-4aec-8b96-b8d09088a6b9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "history = bilal_bert_model_expanded.fit(\n",
        "    [train_encodings.input_ids, train_encodings.token_type_ids, train_encodings.attention_mask], \n",
        "    y_train,\n",
        "    validation_data=(\n",
        "        [valid_encodings.input_ids, valid_encodings.token_type_ids, valid_encodings.attention_mask], \n",
        "        y_valid\n",
        "        ),\n",
        "    batch_size=16, \n",
        "    epochs=4\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kRO1wkUK0GOe",
        "outputId": "9e196918-0154-4472-f4f3-f35f97b6d538"
      },
      "execution_count": 14,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/4\n",
            "2947/2947 [==============================] - 874s 296ms/step - loss: 0.5007 - accuracy: 0.7622 - val_loss: 0.4789 - val_accuracy: 0.7692\n",
            "Epoch 2/4\n",
            "2947/2947 [==============================] - 874s 296ms/step - loss: 0.5001 - accuracy: 0.7621 - val_loss: 0.4833 - val_accuracy: 0.7648\n",
            "Epoch 3/4\n",
            "2947/2947 [==============================] - 873s 296ms/step - loss: 0.4989 - accuracy: 0.7640 - val_loss: 0.4817 - val_accuracy: 0.7675\n",
            "Epoch 4/4\n",
            "2947/2947 [==============================] - 873s 296ms/step - loss: 0.4986 - accuracy: 0.7634 - val_loss: 0.4785 - val_accuracy: 0.7697\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score = bilal_bert_model_expanded.evaluate([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask], y_test)\n",
        "\n",
        "print(\"Test loss:\", score[0])\n",
        "print(\"Test accuracy:\", score[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h1ZdLahp725n",
        "outputId": "129662a6-c2f2-4f3a-900e-6ac61c13c210"
      },
      "execution_count": 15,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "185/185 [==============================] - 81s 435ms/step - loss: 0.4932 - accuracy: 0.7674\n",
            "Test loss: 0.4932304918766022\n",
            "Test accuracy: 0.7673905491828918\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = bilal_bert_model_expanded.predict([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask])\n",
        "preds = predictions.to_tuple()[0].argmax(1)\n",
        "print(classification_report(y_test, preds))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QniCarbm74Ui",
        "outputId": "081a5d26-c715-4f56-9be4-f5eb37b08afc"
      },
      "execution_count": 16,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.78      0.75      0.77      2992\n",
            "           1       0.75      0.78      0.77      2902\n",
            "\n",
            "    accuracy                           0.77      5894\n",
            "   macro avg       0.77      0.77      0.77      5894\n",
            "weighted avg       0.77      0.77      0.77      5894\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "Bilal_et_al_baseline_on_Yelp_data.ipynb",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "80aaf35d190840bf8015f3b63a35c4ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a8d14e05375d489c8f0787e536a1782c",
              "IPY_MODEL_26b05e65a22340c59e17994c4c932028",
              "IPY_MODEL_96977700202e4583b0fcc4d6776a69ce"
            ],
            "layout": "IPY_MODEL_be58acda3e3f4712bcb72bbfc3b338e2"
          }
        },
        "a8d14e05375d489c8f0787e536a1782c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_239dd327934d4d72b2903628a66d556b",
            "placeholder": "​",
            "style": "IPY_MODEL_e491401166184f619e0f8955f528146d",
            "value": "Downloading: 100%"
          }
        },
        "26b05e65a22340c59e17994c4c932028": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49b9c3d055bb45d79245ac97655d3fe3",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bb070356766a42248b882448599da076",
            "value": 231508
          }
        },
        "96977700202e4583b0fcc4d6776a69ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d8ec0c78cfa144c5aeee550a58dc93ba",
            "placeholder": "​",
            "style": "IPY_MODEL_6ad93e985d4f4ff1ada966ff6afa866f",
            "value": " 226k/226k [00:00&lt;00:00, 617kB/s]"
          }
        },
        "be58acda3e3f4712bcb72bbfc3b338e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "239dd327934d4d72b2903628a66d556b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e491401166184f619e0f8955f528146d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "49b9c3d055bb45d79245ac97655d3fe3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb070356766a42248b882448599da076": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d8ec0c78cfa144c5aeee550a58dc93ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ad93e985d4f4ff1ada966ff6afa866f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6505fb018b3b4a0ea1012463f82df3ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d24e921fbe514544a433889da4f421f8",
              "IPY_MODEL_e109fca25373425091ce0fa33fa2d19c",
              "IPY_MODEL_1611fe24ee014a4797b3559d73ab4630"
            ],
            "layout": "IPY_MODEL_bfe89e218bca4b50810e45464e1d986d"
          }
        },
        "d24e921fbe514544a433889da4f421f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b486d2bbacb4060af870dd55fb7c283",
            "placeholder": "​",
            "style": "IPY_MODEL_e7580bf4b52c44a09e1e304e07adec11",
            "value": "Downloading: 100%"
          }
        },
        "e109fca25373425091ce0fa33fa2d19c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c976d7bdb29d416185262635c550a35c",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_64244d3f83e34375b56c2ca460d04bf5",
            "value": 28
          }
        },
        "1611fe24ee014a4797b3559d73ab4630": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68c3ec9c4d0d49ef96b08e46854e88c5",
            "placeholder": "​",
            "style": "IPY_MODEL_23ebc1b675d14aba808674566807cfc0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 865B/s]"
          }
        },
        "bfe89e218bca4b50810e45464e1d986d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b486d2bbacb4060af870dd55fb7c283": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7580bf4b52c44a09e1e304e07adec11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c976d7bdb29d416185262635c550a35c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64244d3f83e34375b56c2ca460d04bf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "68c3ec9c4d0d49ef96b08e46854e88c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23ebc1b675d14aba808674566807cfc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "62c92f8225db48709bfa43f8274a68ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_86d44229d4cb4debb4052cd317bcb4c4",
              "IPY_MODEL_4a3079c3a155423d94bbabf91e7867c5",
              "IPY_MODEL_9747aef346124bcf849cafa15718dbf7"
            ],
            "layout": "IPY_MODEL_e44b238365204e4eb735f34256850aff"
          }
        },
        "86d44229d4cb4debb4052cd317bcb4c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e208b5afaf9403eafb7ba4103b93741",
            "placeholder": "​",
            "style": "IPY_MODEL_18fd148e30844b389970e37a2af2722b",
            "value": "Downloading: 100%"
          }
        },
        "4a3079c3a155423d94bbabf91e7867c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1e96b8685d494462b721deb58ab819c3",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bb6334152a84438d810525fa2c158e3a",
            "value": 570
          }
        },
        "9747aef346124bcf849cafa15718dbf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_46cdab10ea3e4a0d8442baeea4da4e06",
            "placeholder": "​",
            "style": "IPY_MODEL_a499917149ee4a45bfcfa71ea229a839",
            "value": " 570/570 [00:00&lt;00:00, 17.1kB/s]"
          }
        },
        "e44b238365204e4eb735f34256850aff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e208b5afaf9403eafb7ba4103b93741": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18fd148e30844b389970e37a2af2722b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1e96b8685d494462b721deb58ab819c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb6334152a84438d810525fa2c158e3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "46cdab10ea3e4a0d8442baeea4da4e06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a499917149ee4a45bfcfa71ea229a839": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0c7c35d9835431b9a0811dfee57bac0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_239747c48e7941d98d0c39ad65c1300e",
              "IPY_MODEL_e42fd8bf70e3491e80bce582e6769f20",
              "IPY_MODEL_b53fbf0590c948dc86c21b3053f19d3b"
            ],
            "layout": "IPY_MODEL_241d5d1d76064fe89ebe24152da6d486"
          }
        },
        "239747c48e7941d98d0c39ad65c1300e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_288a1c26e2b04ce0b1ffe839cf63cf66",
            "placeholder": "​",
            "style": "IPY_MODEL_35d769c50be7437683162890085cc428",
            "value": "Downloading: 100%"
          }
        },
        "e42fd8bf70e3491e80bce582e6769f20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56d4687fc2f4441fa0a977cbc87b15b6",
            "max": 536063208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6efec14161ee40abb152c7d089cd8b8e",
            "value": 536063208
          }
        },
        "b53fbf0590c948dc86c21b3053f19d3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c918b4ecebac4668b686809b0debc294",
            "placeholder": "​",
            "style": "IPY_MODEL_aa12a5021ab747b9afdd35189ae93217",
            "value": " 511M/511M [00:14&lt;00:00, 34.8MB/s]"
          }
        },
        "241d5d1d76064fe89ebe24152da6d486": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "288a1c26e2b04ce0b1ffe839cf63cf66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "35d769c50be7437683162890085cc428": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "56d4687fc2f4441fa0a977cbc87b15b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6efec14161ee40abb152c7d089cd8b8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c918b4ecebac4668b686809b0debc294": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa12a5021ab747b9afdd35189ae93217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}