.ipynb_checkpoints/BERT-checkpoint.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "pv6KDdCLqngu"
      },
      "outputs": [],
      "source": [
        "MAX_LEN = 512\n",
        "BERT_MODEL = \"bert-base-uncased\"\n",
        "# reduce lr on plateau"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ODpHxEA0vkd7"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "import torch.nn as nn\n",
        "from torch.nn import CrossEntropyLoss\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "\n",
        "\n",
        "import transformers\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import precision_score, recall_score, f1_score\n",
        "import re\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "G2QXzlNkvkd9"
      },
      "outputs": [],
      "source": [
        "class BERTModel(nn.Module):\n",
        "    \"\"\"\n",
        "        BERT model with a dropout and linear layer with 2 outputs\n",
        "    \"\"\"\n",
        "    def __init__(self, bert_model, num_classes=2, dropout_rate=0.3):\n",
        "        super(BERTModel, self).__init__()\n",
        "        self.bert = transformers.BertModel.from_pretrained(bert_model)\n",
        "        self.drop = nn.Dropout(dropout_rate)\n",
        "        self.out = nn.Linear(self.bert.config.hidden_size, num_classes)\n",
        "\n",
        "\n",
        "    def forward(self, ids, mask, token_type_ids):\n",
        "        _, pooled_output = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids, return_dict=False)\n",
        "        output = self.drop(pooled_output)\n",
        "        return self.out(output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "4ykvSgarp9A4"
      },
      "outputs": [],
      "source": [
        "def remove_tags(text):\n",
        "    if isinstance(text, list):\n",
        "        return [remove_tags(t) for t in text]\n",
        "    else:\n",
        "        text = text.lower()\n",
        "        # Remove hashtags\n",
        "        text = re.sub(r'#\\w+', '', text)\n",
        "        # Remove '<user>'\n",
        "        text = re.sub(r'<user>', '', text)\n",
        "        # Remove '<url>'\n",
        "        text = re.sub(r'<url>', '', text)\n",
        "        # remove number\n",
        "        text = re.sub(r'\\d+', '', text)\n",
        "        return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "f3qCUn8gvkd9"
      },
      "outputs": [],
      "source": [
        "def train(data, model, optimizer, device):\n",
        "    \"\"\"\n",
        "        Train the model for one epoch\n",
        "    \"\"\"\n",
        "    model.train()\n",
        "    running_loss = 0.0\n",
        "\n",
        "    for batch_idx, d in enumerate(data):\n",
        "        mask = d['mask'].to(device, dtype=torch.long)\n",
        "        ids = d['ids'].to(device, dtype=torch.long)\n",
        "        token_type_ids = d['token_type_ids'].to(device, dtype=torch.long)\n",
        "        targets = d['targets'].to(device, dtype=torch.long)\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(ids, mask, token_type_ids)\n",
        "\n",
        "        loss = torch.nn.CrossEntropyLoss(outputs, targets) # Calculate loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if batch_idx % 10 == 0 and batch_idx !=0:\n",
        "            temp = f'Batch index = {batch_idx}\\tRunning Loss = {running_loss/10}'\n",
        "            print(temp)\n",
        "            running_loss = 0.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "39defc5139e24431ac20c3f1bc7faa7f",
            "aedb6b756d5949b897c8efcd38622855",
            "55adf2e4079240c28006e6b2f5247c2d",
            "909aa314910d429b9a24e6b33ba41dc8",
            "bacc9e20d09d4064b8d9824d37a43d44",
            "3bd9188ae6414fe3b28f52d88aa6012c",
            "bd417b8d4a8745b293956f05ba48f7b2",
            "0f38fc12874446abbf2767a04e1e05ef",
            "8dd318a1fa2843cbab7047bc3741296e",
            "b679171ab707492ab9d0aebc07232d92",
            "125fc3defb934484b47c9d86cd632bbc",
            "2cb7f33f73a44212903dacb9fc13a5c8",
            "408fe317813f421bb77c97ec30e41125",
            "5f56a9f04032416995d83ebc7e8067b7",
            "399404ae78514d87af805eeae52ac6da",
            "dc8be3d30f5b47ffae3e394102a7392c",
            "9278eac27dc740cfafd944b71bc44eeb",
            "c4c6b654729b4221962a4fdc30bbcb9e",
            "9aeee5e49361449b8a269519a4f176d0",
            "175368bb903c41b6b9dc09574b2f27d6",
            "73f44759a18d43229dcadc240334944f",
            "36bf77fe27ab483997e073c49096464a",
            "726d5e2f329b4d12a8088b6667ea42c0",
            "ab80dce52b374653b0d363e821b28468",
            "79e3872c7ffd4fc19602cd0917243da3",
            "b347db1d29654fd9b096b789d8b6080a",
            "8a296714b44044689d1881fd9ec84160",
            "eed58ea3c0e942d4a93f98080f1ea808",
            "bd099c2b893244eb8c07824b3350a48f",
            "72353c6129d44c2ab0654b922fd9a0bb",
            "f86cfe1ddaa240ce8eaac38a82c20abd",
            "9156a6bc795f42beb5a5991b23591033",
            "443e61c993dd42eb9c9ad782b11eca93",
            "f2b5b740966544598f468398a42121c6",
            "89e4a680539f47a3ab8a6ff018898df4",
            "9d6c22f05b2f4501bea23c38edcbbb6b",
            "0fa78a342be94571a268ee61f4e3c01d",
            "2fa8386c7a7b47f4a5417002e5fec63a",
            "0b5d6a1e607144878554a64ebab0533f",
            "ae76992fb23943f7ac30b058a4c8e16a",
            "636e173a4d8a4d5db8b34b8935db62e0",
            "26d819d3a7234c05978fa8b38a3d3c61",
            "9094bfe693954583927f67e355770eaf",
            "4d26a53fe26549349e45e227c1bfd585"
          ]
        },
        "id": "6quHUiImvkd9",
        "outputId": "a1796ed9-7386-487c-fddb-9975d58bbd79"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'str'>\n"
          ]
        }
      ],
      "source": [
        "pos_path = 'data/twitter-datasets/train_pos.txt'\n",
        "neg_path = 'data/twitter-datasets/train_neg.txt'\n",
        "\n",
        "with open(pos_path, 'r') as f:\n",
        "    pos_tweets = f.readlines()\n",
        "with open(neg_path, 'r') as f:\n",
        "    neg_tweets = f.readlines()\n",
        "\n",
        "# TODO: preprocess data\n",
        "pos_tweets = remove_tags(pos_tweets)\n",
        "neg_tweets = remove_tags(neg_tweets)\n",
        "\n",
        "\n",
        "pos_labels = [1 for _ in range(len(pos_tweets))]\n",
        "neg_labels = [0 for _ in range(len(neg_tweets))]\n",
        "labels = pos_labels + neg_labels\n",
        "tweets = pos_tweets + neg_tweets\n",
        "\n",
        "train_tweets, val_tweets, train_labels, val_labels = train_test_split(tweets, labels, test_size=0.1)\n",
        "\n",
        "# tokenizer = BertTokenizer.from_pretrained(BERT_MODEL, do_lower_case=True)\n",
        "# train_encodings = tokenizer(train_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
        "# val_encodings = tokenizer(val_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
        "\n",
        "tokenizer = TweetTokenizer()\n",
        "train_encodings = [tokenizer.tokenize(t) for t in train_tweets]\n",
        "val_encodings = [tokenizer.tokenize(t) for t in val_tweets]\n",
        "\n",
        "\n",
        "class TweetDataset(Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_dataset = TweetDataset(train_encodings, train_labels)\n",
        "val_dataset = TweetDataset(val_encodings, val_labels)\n",
        "train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
        "val_loader = DataLoader(val_dataset, batch_size=8, shuffle=False)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "dbdf4e8013ca45f78e131323c7437659",
            "02abcd45b1864d078cc550c667fdaf48",
            "7871b157ae36445cac915f7fe44f5856",
            "03b02aed55704de489049ff99a8794c3",
            "88a9c0743a344f3bb06c7c411730822d",
            "d2989b8f0d1e4896b6f348dd4dddfe2b",
            "102153cc6f3248949ab7ac5875b73907",
            "28c1be7267954ebd9b0811bb128016f8",
            "784f5867029247ecacfb9cf7912b490a",
            "ff749937033648f9a17b5f36a31d9765",
            "1d758595b9c04c4c8b5d097b05683d7a"
          ]
        },
        "id": "hhA0rrcSvkd-",
        "outputId": "aeaf57dc-8a80-44e4-a5c5-bed3a9eb60d6"
      },
      "outputs": [],
      "source": [
        "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
        "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
        "num_epochs = 3  # Number of training epochs\n",
        "\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model.to(device)\n",
        "criterion = CrossEntropyLoss()\n",
        "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps=0,\n",
        "                                            num_training_steps=len(train_loader) * num_epochs)\n",
        "\n",
        "\n",
        "# Function to calculate accuracy\n",
        "def calc_accuracy(preds, labels):\n",
        "    _, predictions = torch.max(preds, dim=1)\n",
        "    correct = (predictions == labels).float()\n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc\n",
        "\n",
        "all_labels = []\n",
        "all_predictions = []\n",
        "\n",
        "# Training Loop\n",
        "for epoch in range(num_epochs):\n",
        "    print(f\"Epoch {epoch+1}/{num_epochs}\")\n",
        "    model.train()\n",
        "    total_train_loss = 0\n",
        "    total_train_acc = 0\n",
        "\n",
        "    for step, batch in enumerate(train_loader):\n",
        "        batch = {k: v.to(device) for k, v in batch.items()}\n",
        "        outputs = model(**batch)\n",
        "        loss = criterion(outputs.logits, batch['labels'])\n",
        "        acc = calc_accuracy(outputs.logits, batch['labels'])\n",
        "        print(f\"Step {step+1}/{len(train_loader)} - Loss: {loss.item()}, Accuracy: {acc.item()}\")\n",
        "        total_train_loss += loss.item()\n",
        "        total_train_acc += acc.item()\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_loader)\n",
        "    avg_train_acc = total_train_acc / len(train_loader)\n",
        "    print(f\"Epoch {epoch+1} - Train loss: {avg_train_loss}, Accuracy: {avg_train_acc}\")\n",
        "\n",
        "    # Validation Loop\n",
        "    model.eval()\n",
        "    total_val_accuracy = 0\n",
        "    total_val_loss = 0\n",
        "\n",
        "    for batch in val_loader:\n",
        "        with torch.no_grad():\n",
        "            batch = {k: v.to(device) for k, v in batch.items()}\n",
        "            outputs = model(**batch)\n",
        "            loss = criterion(outputs.logits, batch['labels'])\n",
        "            acc = calc_accuracy(outputs.logits, batch['labels'])\n",
        "\n",
        "            total_val_loss += loss.item()\n",
        "            total_val_accuracy += acc.item()\n",
        "\n",
        "            # for overall metrics calculation\n",
        "            preds = outputs.logits.argmax(dim=1).cpu().numpy()\n",
        "            labels = batch['labels'].cpu().numpy()\n",
        "            all_predictions.extend(preds)\n",
        "            all_labels.extend(labels)\n",
        "\n",
        "    avg_val_loss = total_val_loss / len(val_loader)\n",
        "    avg_val_accuracy = total_val_accuracy / len(val_loader)\n",
        "    print(f\"Epoch {epoch+1} - Validation loss: {avg_val_loss}, Accuracy: {avg_val_accuracy}\")\n",
        "\n",
        "# After all epochs, calculate overall metrics\n",
        "precision = precision_score(all_labels, all_predictions, average='macro')\n",
        "recall = recall_score(all_labels, all_predictions, average='macro')\n",
        "f1 = f1_score(all_labels, all_predictions, average='macro')\n",
        "\n",
        "print(\"Training complete!\")\n",
        "print(f\"Overall Precision: {precision}, Recall: {recall}, F1 Score: {f1}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Possible improvement\n",
        "1. Don't store all the scores in one variable, just store for each epoch and only keep the average score\n",
        "2. save model for Each epoch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1phcjpJ7vkd-"
      },
      "outputs": [],
      "source": [
        "torch.save(model, 'bert.pth')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Kvt6507Ha_sp"
      },
      "outputs": [],
      "source": [
        "## predict\n",
        "model = torch.load('bert.pth')\n",
        "\n",
        "## use the test set\n",
        "test_path = 'data/twitter-datasets/test_data.txt'\n",
        "with open(test_path, 'r') as f:\n",
        "    test_tweets = f.readlines()\n",
        "\n",
        "test_encodings = tokenizer(test_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
        "test_dataset = TweetDataset(test_encodings, [0 for _ in range(len(test_tweets))])\n",
        "test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False)\n",
        "\n",
        "\n",
        "model.eval()\n",
        "predictions = []\n",
        "for batch in test_loader:\n",
        "    with torch.no_grad():\n",
        "        batch = {k: v.to(device) for k, v in batch.items()}\n",
        "        outputs = model(**batch)\n",
        "        preds = outputs.logits.argmax(dim=1).cpu().numpy()\n",
        "        predictions.extend(preds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b6r6SXzwbgee"
      },
      "outputs": [],
      "source": [
        "predictions = np.array(predictions)\n",
        "predictions[predictions == 0] = -1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m_iHW_xrbVlm",
        "outputId": "1b26e777-3a61-46fa-af39-f45a26130da2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[-1 -1  1 ... -1  1 -1]\n"
          ]
        }
      ],
      "source": [
        "print(predictions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zG1C1sVVbKZp"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "submission = pd.DataFrame({'Id':range(1, len(predictions) + 1),'Prediction': predictions})\n",
        "submission.to_csv('submission.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.18"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02abcd45b1864d078cc550c667fdaf48": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2989b8f0d1e4896b6f348dd4dddfe2b",
            "placeholder": "​",
            "style": "IPY_MODEL_102153cc6f3248949ab7ac5875b73907",
            "value": "model.safetensors: 100%"
          }
        },
        "03b02aed55704de489049ff99a8794c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff749937033648f9a17b5f36a31d9765",
            "placeholder": "​",
            "style": "IPY_MODEL_1d758595b9c04c4c8b5d097b05683d7a",
            "value": " 440M/440M [00:01&lt;00:00, 341MB/s]"
          }
        },
        "0b5d6a1e607144878554a64ebab0533f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f38fc12874446abbf2767a04e1e05ef": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0fa78a342be94571a268ee61f4e3c01d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9094bfe693954583927f67e355770eaf",
            "placeholder": "​",
            "style": "IPY_MODEL_4d26a53fe26549349e45e227c1bfd585",
            "value": " 570/570 [00:00&lt;00:00, 43.6kB/s]"
          }
        },
        "102153cc6f3248949ab7ac5875b73907": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "125fc3defb934484b47c9d86cd632bbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "175368bb903c41b6b9dc09574b2f27d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d758595b9c04c4c8b5d097b05683d7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "26d819d3a7234c05978fa8b38a3d3c61": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "28c1be7267954ebd9b0811bb128016f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cb7f33f73a44212903dacb9fc13a5c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_408fe317813f421bb77c97ec30e41125",
              "IPY_MODEL_5f56a9f04032416995d83ebc7e8067b7",
              "IPY_MODEL_399404ae78514d87af805eeae52ac6da"
            ],
            "layout": "IPY_MODEL_dc8be3d30f5b47ffae3e394102a7392c"
          }
        },
        "2fa8386c7a7b47f4a5417002e5fec63a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36bf77fe27ab483997e073c49096464a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "399404ae78514d87af805eeae52ac6da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73f44759a18d43229dcadc240334944f",
            "placeholder": "​",
            "style": "IPY_MODEL_36bf77fe27ab483997e073c49096464a",
            "value": " 232k/232k [00:00&lt;00:00, 473kB/s]"
          }
        },
        "39defc5139e24431ac20c3f1bc7faa7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aedb6b756d5949b897c8efcd38622855",
              "IPY_MODEL_55adf2e4079240c28006e6b2f5247c2d",
              "IPY_MODEL_909aa314910d429b9a24e6b33ba41dc8"
            ],
            "layout": "IPY_MODEL_bacc9e20d09d4064b8d9824d37a43d44"
          }
        },
        "3bd9188ae6414fe3b28f52d88aa6012c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "408fe317813f421bb77c97ec30e41125": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9278eac27dc740cfafd944b71bc44eeb",
            "placeholder": "​",
            "style": "IPY_MODEL_c4c6b654729b4221962a4fdc30bbcb9e",
            "value": "vocab.txt: 100%"
          }
        },
        "443e61c993dd42eb9c9ad782b11eca93": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4d26a53fe26549349e45e227c1bfd585": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55adf2e4079240c28006e6b2f5247c2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f38fc12874446abbf2767a04e1e05ef",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8dd318a1fa2843cbab7047bc3741296e",
            "value": 28
          }
        },
        "5f56a9f04032416995d83ebc7e8067b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9aeee5e49361449b8a269519a4f176d0",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_175368bb903c41b6b9dc09574b2f27d6",
            "value": 231508
          }
        },
        "636e173a4d8a4d5db8b34b8935db62e0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72353c6129d44c2ab0654b922fd9a0bb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "726d5e2f329b4d12a8088b6667ea42c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ab80dce52b374653b0d363e821b28468",
              "IPY_MODEL_79e3872c7ffd4fc19602cd0917243da3",
              "IPY_MODEL_b347db1d29654fd9b096b789d8b6080a"
            ],
            "layout": "IPY_MODEL_8a296714b44044689d1881fd9ec84160"
          }
        },
        "73f44759a18d43229dcadc240334944f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "784f5867029247ecacfb9cf7912b490a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7871b157ae36445cac915f7fe44f5856": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28c1be7267954ebd9b0811bb128016f8",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_784f5867029247ecacfb9cf7912b490a",
            "value": 440449768
          }
        },
        "79e3872c7ffd4fc19602cd0917243da3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72353c6129d44c2ab0654b922fd9a0bb",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f86cfe1ddaa240ce8eaac38a82c20abd",
            "value": 466062
          }
        },
        "88a9c0743a344f3bb06c7c411730822d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89e4a680539f47a3ab8a6ff018898df4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b5d6a1e607144878554a64ebab0533f",
            "placeholder": "​",
            "style": "IPY_MODEL_ae76992fb23943f7ac30b058a4c8e16a",
            "value": "config.json: 100%"
          }
        },
        "8a296714b44044689d1881fd9ec84160": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dd318a1fa2843cbab7047bc3741296e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9094bfe693954583927f67e355770eaf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "909aa314910d429b9a24e6b33ba41dc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b679171ab707492ab9d0aebc07232d92",
            "placeholder": "​",
            "style": "IPY_MODEL_125fc3defb934484b47c9d86cd632bbc",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.76kB/s]"
          }
        },
        "9156a6bc795f42beb5a5991b23591033": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9278eac27dc740cfafd944b71bc44eeb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9aeee5e49361449b8a269519a4f176d0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d6c22f05b2f4501bea23c38edcbbb6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_636e173a4d8a4d5db8b34b8935db62e0",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_26d819d3a7234c05978fa8b38a3d3c61",
            "value": 570
          }
        },
        "ab80dce52b374653b0d363e821b28468": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eed58ea3c0e942d4a93f98080f1ea808",
            "placeholder": "​",
            "style": "IPY_MODEL_bd099c2b893244eb8c07824b3350a48f",
            "value": "tokenizer.json: 100%"
          }
        },
        "ae76992fb23943f7ac30b058a4c8e16a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aedb6b756d5949b897c8efcd38622855": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bd9188ae6414fe3b28f52d88aa6012c",
            "placeholder": "​",
            "style": "IPY_MODEL_bd417b8d4a8745b293956f05ba48f7b2",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "b347db1d29654fd9b096b789d8b6080a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9156a6bc795f42beb5a5991b23591033",
            "placeholder": "​",
            "style": "IPY_MODEL_443e61c993dd42eb9c9ad782b11eca93",
            "value": " 466k/466k [00:00&lt;00:00, 1.90MB/s]"
          }
        },
        "b679171ab707492ab9d0aebc07232d92": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bacc9e20d09d4064b8d9824d37a43d44": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd099c2b893244eb8c07824b3350a48f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd417b8d4a8745b293956f05ba48f7b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4c6b654729b4221962a4fdc30bbcb9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d2989b8f0d1e4896b6f348dd4dddfe2b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbdf4e8013ca45f78e131323c7437659": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_02abcd45b1864d078cc550c667fdaf48",
              "IPY_MODEL_7871b157ae36445cac915f7fe44f5856",
              "IPY_MODEL_03b02aed55704de489049ff99a8794c3"
            ],
            "layout": "IPY_MODEL_88a9c0743a344f3bb06c7c411730822d"
          }
        },
        "dc8be3d30f5b47ffae3e394102a7392c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eed58ea3c0e942d4a93f98080f1ea808": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2b5b740966544598f468398a42121c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_89e4a680539f47a3ab8a6ff018898df4",
              "IPY_MODEL_9d6c22f05b2f4501bea23c38edcbbb6b",
              "IPY_MODEL_0fa78a342be94571a268ee61f4e3c01d"
            ],
            "layout": "IPY_MODEL_2fa8386c7a7b47f4a5417002e5fec63a"
          }
        },
        "f86cfe1ddaa240ce8eaac38a82c20abd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ff749937033648f9a17b5f36a31d9765": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}