.ipynb_checkpoints/Twitter_roberta-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "pv6KDdCLqngu"
   },
   "outputs": [],
   "source": [
    "MAX_LEN = 512\n",
    "TWITTER_ROBERTA_MODEL = f\"cardiffnlp/twitter-roberta-base-sentiment-latest\"\n",
    "# reduce lr on plateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "ODpHxEA0vkd7"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import torch.nn as nn\n",
    "from torch.nn import CrossEntropyLoss\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoConfig\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import TFAutoModelForSequenceClassification\n",
    "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "G2QXzlNkvkd9"
   },
   "outputs": [],
   "source": [
    "class BERTModel(nn.Module):\n",
    "    \"\"\"\n",
    "        BERT model with a dropout and linear layer with 2 outputs\n",
    "    \"\"\"\n",
    "    def __init__(self, bert_model, num_classes=2, dropout_rate=0.3):\n",
    "        super(BERTModel, self).__init__()\n",
    "        self.bert = AutoTokenizer.from_pretrained(TWITTER_ROBERTA_MODEL)\n",
    "        self.drop = nn.Dropout(dropout_rate)\n",
    "        self.out = nn.Linear(self.bert.config.hidden_size, num_classes)\n",
    "\n",
    "\n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, pooled_output = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids, return_dict=False)\n",
    "        output = self.drop(pooled_output)\n",
    "        return self.out(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "4ykvSgarp9A4"
   },
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    if isinstance(text, list):\n",
    "        return [remove_tags(t) for t in text]\n",
    "    else:\n",
    "        # Remove '<user>'\n",
    "        text = re.sub(r'<user>', '', text)\n",
    "        # Remove '<url>'\n",
    "        text = re.sub(r'<url>', '', text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "f3qCUn8gvkd9"
   },
   "outputs": [],
   "source": [
    "def train(data, model, optimizer, device):\n",
    "    \"\"\"\n",
    "        Train the model for one epoch\n",
    "    \"\"\"\n",
    "    model.train()\n",
    "    running_loss = 0.0\n",
    "\n",
    "    for batch_idx, d in enumerate(data):\n",
    "        mask = d['mask'].to(device, dtype=torch.long)\n",
    "        ids = d['ids'].to(device, dtype=torch.long)\n",
    "        token_type_ids = d['token_type_ids'].to(device, dtype=torch.long)\n",
    "        targets = d['targets'].to(device, dtype=torch.long)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "        loss = torch.nn.CrossEntropyLoss(outputs, targets) # Calculate loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if batch_idx % 10 == 0 and batch_idx !=0:\n",
    "            temp = f'Batch index = {batch_idx}\\tRunning Loss = {running_loss/10}'\n",
    "            print(temp)\n",
    "            running_loss = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145,
     "referenced_widgets": [
      "39defc5139e24431ac20c3f1bc7faa7f",
      "aedb6b756d5949b897c8efcd38622855",
      "55adf2e4079240c28006e6b2f5247c2d",
      "909aa314910d429b9a24e6b33ba41dc8",
      "bacc9e20d09d4064b8d9824d37a43d44",
      "3bd9188ae6414fe3b28f52d88aa6012c",
      "bd417b8d4a8745b293956f05ba48f7b2",
      "0f38fc12874446abbf2767a04e1e05ef",
      "8dd318a1fa2843cbab7047bc3741296e",
      "b679171ab707492ab9d0aebc07232d92",
      "125fc3defb934484b47c9d86cd632bbc",
      "2cb7f33f73a44212903dacb9fc13a5c8",
      "408fe317813f421bb77c97ec30e41125",
      "5f56a9f04032416995d83ebc7e8067b7",
      "399404ae78514d87af805eeae52ac6da",
      "dc8be3d30f5b47ffae3e394102a7392c",
      "9278eac27dc740cfafd944b71bc44eeb",
      "c4c6b654729b4221962a4fdc30bbcb9e",
      "9aeee5e49361449b8a269519a4f176d0",
      "175368bb903c41b6b9dc09574b2f27d6",
      "73f44759a18d43229dcadc240334944f",
      "36bf77fe27ab483997e073c49096464a",
      "726d5e2f329b4d12a8088b6667ea42c0",
      "ab80dce52b374653b0d363e821b28468",
      "79e3872c7ffd4fc19602cd0917243da3",
      "b347db1d29654fd9b096b789d8b6080a",
      "8a296714b44044689d1881fd9ec84160",
      "eed58ea3c0e942d4a93f98080f1ea808",
      "bd099c2b893244eb8c07824b3350a48f",
      "72353c6129d44c2ab0654b922fd9a0bb",
      "f86cfe1ddaa240ce8eaac38a82c20abd",
      "9156a6bc795f42beb5a5991b23591033",
      "443e61c993dd42eb9c9ad782b11eca93",
      "f2b5b740966544598f468398a42121c6",
      "89e4a680539f47a3ab8a6ff018898df4",
      "9d6c22f05b2f4501bea23c38edcbbb6b",
      "0fa78a342be94571a268ee61f4e3c01d",
      "2fa8386c7a7b47f4a5417002e5fec63a",
      "0b5d6a1e607144878554a64ebab0533f",
      "ae76992fb23943f7ac30b058a4c8e16a",
      "636e173a4d8a4d5db8b34b8935db62e0",
      "26d819d3a7234c05978fa8b38a3d3c61",
      "9094bfe693954583927f67e355770eaf",
      "4d26a53fe26549349e45e227c1bfd585"
     ]
    },
    "id": "6quHUiImvkd9",
    "outputId": "a1796ed9-7386-487c-fddb-9975d58bbd79"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'>\n"
     ]
    }
   ],
   "source": [
    "pos_path = 'data/twitter-datasets/train_pos.txt'\n",
    "neg_path = 'data/twitter-datasets/train_neg.txt'\n",
    "\n",
    "with open(pos_path, 'r') as f:\n",
    "    pos_tweets = f.readlines()\n",
    "with open(neg_path, 'r') as f:\n",
    "    neg_tweets = f.readlines()\n",
    "\n",
    "# TODO: preprocess data\n",
    "pos_tweets = preprocess(pos_tweets)\n",
    "neg_tweets = preprocess(neg_tweets)\n",
    "\n",
    "\n",
    "pos_labels = [1 for _ in range(len(pos_tweets))]\n",
    "neg_labels = [0 for _ in range(len(neg_tweets))]\n",
    "labels = pos_labels + neg_labels\n",
    "tweets = pos_tweets + neg_tweets\n",
    "\n",
    "train_tweets, val_tweets, train_labels, val_labels = train_test_split(tweets, labels, test_size=0.1)\n",
    "\n",
    "# tokenizer = BertTokenizer.from_pretrained(BERT_MODEL, do_lower_case=True)\n",
    "# train_encodings = tokenizer(train_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
    "# val_encodings = tokenizer(val_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(TWITTER_ROBERTA_MODEL)\n",
    "train_encodings = [tokenizer.tokenize(t) for t in train_tweets]\n",
    "val_encodings = [tokenizer.tokenize(t) for t in val_tweets]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TweetDataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = TweetDataset(train_encodings, train_labels)\n",
    "val_dataset = TweetDataset(val_encodings, val_labels)\n",
    "train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "dbdf4e8013ca45f78e131323c7437659",
      "02abcd45b1864d078cc550c667fdaf48",
      "7871b157ae36445cac915f7fe44f5856",
      "03b02aed55704de489049ff99a8794c3",
      "88a9c0743a344f3bb06c7c411730822d",
      "d2989b8f0d1e4896b6f348dd4dddfe2b",
      "102153cc6f3248949ab7ac5875b73907",
      "28c1be7267954ebd9b0811bb128016f8",
      "784f5867029247ecacfb9cf7912b490a",
      "ff749937033648f9a17b5f36a31d9765",
      "1d758595b9c04c4c8b5d097b05683d7a"
     ]
    },
    "id": "hhA0rrcSvkd-",
    "outputId": "aeaf57dc-8a80-44e4-a5c5-bed3a9eb60d6"
   },
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(TWITTER_ROBERTA_MODEL)\n",
    "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
    "num_epochs = 3  # Number of training epochs\n",
    "\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "model.to(device)\n",
    "criterion = CrossEntropyLoss()\n",
    "optimizer = AdamW(model.parameters(), lr=5e-5)\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
    "                                            num_warmup_steps=0,\n",
    "                                            num_training_steps=len(train_loader) * num_epochs)\n",
    "\n",
    "\n",
    "# Function to calculate accuracy\n",
    "def calc_accuracy(preds, labels):\n",
    "    _, predictions = torch.max(preds, dim=1)\n",
    "    correct = (predictions == labels).float()\n",
    "    acc = correct.sum() / len(correct)\n",
    "    return acc\n",
    "\n",
    "all_labels = []\n",
    "all_predictions = []\n",
    "\n",
    "# Training Loop\n",
    "for epoch in range(num_epochs):\n",
    "    print(f\"Epoch {epoch+1}/{num_epochs}\")\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    total_train_acc = 0\n",
    "\n",
    "    for step, batch in enumerate(train_loader):\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        outputs = model(**batch)\n",
    "        loss = criterion(outputs.logits, batch['labels'])\n",
    "        acc = calc_accuracy(outputs.logits, batch['labels'])\n",
    "        print(f\"Step {step+1}/{len(train_loader)} - Loss: {loss.item()}, Accuracy: {acc.item()}\")\n",
    "        total_train_loss += loss.item()\n",
    "        total_train_acc += acc.item()\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "    avg_train_loss = total_train_loss / len(train_loader)\n",
    "    avg_train_acc = total_train_acc / len(train_loader)\n",
    "    print(f\"Epoch {epoch+1} - Train loss: {avg_train_loss}, Accuracy: {avg_train_acc}\")\n",
    "\n",
    "    # Validation Loop\n",
    "    model.eval()\n",
    "    total_val_accuracy = 0\n",
    "    total_val_loss = 0\n",
    "\n",
    "    for batch in val_loader:\n",
    "        with torch.no_grad():\n",
    "            batch = {k: v.to(device) for k, v in batch.items()}\n",
    "            outputs = model(**batch)\n",
    "            loss = criterion(outputs.logits, batch['labels'])\n",
    "            acc = calc_accuracy(outputs.logits, batch['labels'])\n",
    "\n",
    "            total_val_loss += loss.item()\n",
    "            total_val_accuracy += acc.item()\n",
    "\n",
    "            # for overall metrics calculation\n",
    "            preds = outputs.logits.argmax(dim=1).cpu().numpy()\n",
    "            labels = batch['labels'].cpu().numpy()\n",
    "            all_predictions.extend(preds)\n",
    "            all_labels.extend(labels)\n",
    "\n",
    "    avg_val_loss = total_val_loss / len(val_loader)\n",
    "    avg_val_accuracy = total_val_accuracy / len(val_loader)\n",
    "    print(f\"Epoch {epoch+1} - Validation loss: {avg_val_loss}, Accuracy: {avg_val_accuracy}\")\n",
    "\n",
    "# After all epochs, calculate overall metrics\n",
    "precision = precision_score(all_labels, all_predictions, average='macro')\n",
    "recall = recall_score(all_labels, all_predictions, average='macro')\n",
    "f1 = f1_score(all_labels, all_predictions, average='macro')\n",
    "\n",
    "print(\"Training complete!\")\n",
    "print(f\"Overall Precision: {precision}, Recall: {recall}, F1 Score: {f1}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Possible improvement\n",
    "1. Don't store all the scores in one variable, just store for each epoch and only keep the average score\n",
    "2. save model for Each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1phcjpJ7vkd-"
   },
   "outputs": [],
   "source": [
    "torch.save(model, 'bert.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Kvt6507Ha_sp"
   },
   "outputs": [],
   "source": [
    "## predict\n",
    "model = torch.load('bert.pth')\n",
    "\n",
    "## use the test set\n",
    "test_path = 'data/twitter-datasets/test_data.txt'\n",
    "with open(test_path, 'r') as f:\n",
    "    test_tweets = f.readlines()\n",
    "\n",
    "test_encodings = tokenizer(test_tweets, truncation=True, padding=True, max_length=MAX_LEN)\n",
    "test_dataset = TweetDataset(test_encodings, [0 for _ in range(len(test_tweets))])\n",
    "test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False)\n",
    "\n",
    "\n",
    "model.eval()\n",
    "predictions = []\n",
    "for batch in test_loader:\n",
    "    with torch.no_grad():\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        outputs = model(**batch)\n",
    "        preds = outputs.logits.argmax(dim=1).cpu().numpy()\n",
    "        predictions.extend(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "b6r6SXzwbgee"
   },
   "outputs": [],
   "source": [
    "predictions = np.array(predictions)\n",
    "predictions[predictions == 0] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "m_iHW_xrbVlm",
    "outputId": "1b26e777-3a61-46fa-af39-f45a26130da2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1 -1  1 ... -1  1 -1]\n"
     ]
    }
   ],
   "source": [
    "print(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zG1C1sVVbKZp"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "submission = pd.DataFrame({'Id':range(1, len(predictions) + 1),'Prediction': predictions})\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "02abcd45b1864d078cc550c667fdaf48": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d2989b8f0d1e4896b6f348dd4dddfe2b",
      "placeholder": "​",
      "style": "IPY_MODEL_102153cc6f3248949ab7ac5875b73907",
      "value": "model.safetensors: 100%"
     }
    },
    "03b02aed55704de489049ff99a8794c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ff749937033648f9a17b5f36a31d9765",
      "placeholder": "​",
      "style": "IPY_MODEL_1d758595b9c04c4c8b5d097b05683d7a",
      "value": " 440M/440M [00:01&lt;00:00, 341MB/s]"
     }
    },
    "0b5d6a1e607144878554a64ebab0533f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0f38fc12874446abbf2767a04e1e05ef": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0fa78a342be94571a268ee61f4e3c01d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9094bfe693954583927f67e355770eaf",
      "placeholder": "​",
      "style": "IPY_MODEL_4d26a53fe26549349e45e227c1bfd585",
      "value": " 570/570 [00:00&lt;00:00, 43.6kB/s]"
     }
    },
    "102153cc6f3248949ab7ac5875b73907": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "125fc3defb934484b47c9d86cd632bbc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "175368bb903c41b6b9dc09574b2f27d6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "1d758595b9c04c4c8b5d097b05683d7a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "26d819d3a7234c05978fa8b38a3d3c61": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "28c1be7267954ebd9b0811bb128016f8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2cb7f33f73a44212903dacb9fc13a5c8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_408fe317813f421bb77c97ec30e41125",
       "IPY_MODEL_5f56a9f04032416995d83ebc7e8067b7",
       "IPY_MODEL_399404ae78514d87af805eeae52ac6da"
      ],
      "layout": "IPY_MODEL_dc8be3d30f5b47ffae3e394102a7392c"
     }
    },
    "2fa8386c7a7b47f4a5417002e5fec63a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "36bf77fe27ab483997e073c49096464a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "399404ae78514d87af805eeae52ac6da": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_73f44759a18d43229dcadc240334944f",
      "placeholder": "​",
      "style": "IPY_MODEL_36bf77fe27ab483997e073c49096464a",
      "value": " 232k/232k [00:00&lt;00:00, 473kB/s]"
     }
    },
    "39defc5139e24431ac20c3f1bc7faa7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_aedb6b756d5949b897c8efcd38622855",
       "IPY_MODEL_55adf2e4079240c28006e6b2f5247c2d",
       "IPY_MODEL_909aa314910d429b9a24e6b33ba41dc8"
      ],
      "layout": "IPY_MODEL_bacc9e20d09d4064b8d9824d37a43d44"
     }
    },
    "3bd9188ae6414fe3b28f52d88aa6012c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "408fe317813f421bb77c97ec30e41125": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9278eac27dc740cfafd944b71bc44eeb",
      "placeholder": "​",
      "style": "IPY_MODEL_c4c6b654729b4221962a4fdc30bbcb9e",
      "value": "vocab.txt: 100%"
     }
    },
    "443e61c993dd42eb9c9ad782b11eca93": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4d26a53fe26549349e45e227c1bfd585": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "55adf2e4079240c28006e6b2f5247c2d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0f38fc12874446abbf2767a04e1e05ef",
      "max": 28,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8dd318a1fa2843cbab7047bc3741296e",
      "value": 28
     }
    },
    "5f56a9f04032416995d83ebc7e8067b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9aeee5e49361449b8a269519a4f176d0",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_175368bb903c41b6b9dc09574b2f27d6",
      "value": 231508
     }
    },
    "636e173a4d8a4d5db8b34b8935db62e0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "72353c6129d44c2ab0654b922fd9a0bb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "726d5e2f329b4d12a8088b6667ea42c0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ab80dce52b374653b0d363e821b28468",
       "IPY_MODEL_79e3872c7ffd4fc19602cd0917243da3",
       "IPY_MODEL_b347db1d29654fd9b096b789d8b6080a"
      ],
      "layout": "IPY_MODEL_8a296714b44044689d1881fd9ec84160"
     }
    },
    "73f44759a18d43229dcadc240334944f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "784f5867029247ecacfb9cf7912b490a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "7871b157ae36445cac915f7fe44f5856": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_28c1be7267954ebd9b0811bb128016f8",
      "max": 440449768,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_784f5867029247ecacfb9cf7912b490a",
      "value": 440449768
     }
    },
    "79e3872c7ffd4fc19602cd0917243da3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_72353c6129d44c2ab0654b922fd9a0bb",
      "max": 466062,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f86cfe1ddaa240ce8eaac38a82c20abd",
      "value": 466062
     }
    },
    "88a9c0743a344f3bb06c7c411730822d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "89e4a680539f47a3ab8a6ff018898df4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0b5d6a1e607144878554a64ebab0533f",
      "placeholder": "​",
      "style": "IPY_MODEL_ae76992fb23943f7ac30b058a4c8e16a",
      "value": "config.json: 100%"
     }
    },
    "8a296714b44044689d1881fd9ec84160": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8dd318a1fa2843cbab7047bc3741296e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "9094bfe693954583927f67e355770eaf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "909aa314910d429b9a24e6b33ba41dc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b679171ab707492ab9d0aebc07232d92",
      "placeholder": "​",
      "style": "IPY_MODEL_125fc3defb934484b47c9d86cd632bbc",
      "value": " 28.0/28.0 [00:00&lt;00:00, 1.76kB/s]"
     }
    },
    "9156a6bc795f42beb5a5991b23591033": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9278eac27dc740cfafd944b71bc44eeb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9aeee5e49361449b8a269519a4f176d0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9d6c22f05b2f4501bea23c38edcbbb6b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_636e173a4d8a4d5db8b34b8935db62e0",
      "max": 570,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_26d819d3a7234c05978fa8b38a3d3c61",
      "value": 570
     }
    },
    "ab80dce52b374653b0d363e821b28468": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eed58ea3c0e942d4a93f98080f1ea808",
      "placeholder": "​",
      "style": "IPY_MODEL_bd099c2b893244eb8c07824b3350a48f",
      "value": "tokenizer.json: 100%"
     }
    },
    "ae76992fb23943f7ac30b058a4c8e16a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "aedb6b756d5949b897c8efcd38622855": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3bd9188ae6414fe3b28f52d88aa6012c",
      "placeholder": "​",
      "style": "IPY_MODEL_bd417b8d4a8745b293956f05ba48f7b2",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "b347db1d29654fd9b096b789d8b6080a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9156a6bc795f42beb5a5991b23591033",
      "placeholder": "​",
      "style": "IPY_MODEL_443e61c993dd42eb9c9ad782b11eca93",
      "value": " 466k/466k [00:00&lt;00:00, 1.90MB/s]"
     }
    },
    "b679171ab707492ab9d0aebc07232d92": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bacc9e20d09d4064b8d9824d37a43d44": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bd099c2b893244eb8c07824b3350a48f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bd417b8d4a8745b293956f05ba48f7b2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c4c6b654729b4221962a4fdc30bbcb9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d2989b8f0d1e4896b6f348dd4dddfe2b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dbdf4e8013ca45f78e131323c7437659": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_02abcd45b1864d078cc550c667fdaf48",
       "IPY_MODEL_7871b157ae36445cac915f7fe44f5856",
       "IPY_MODEL_03b02aed55704de489049ff99a8794c3"
      ],
      "layout": "IPY_MODEL_88a9c0743a344f3bb06c7c411730822d"
     }
    },
    "dc8be3d30f5b47ffae3e394102a7392c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "eed58ea3c0e942d4a93f98080f1ea808": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f2b5b740966544598f468398a42121c6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_89e4a680539f47a3ab8a6ff018898df4",
       "IPY_MODEL_9d6c22f05b2f4501bea23c38edcbbb6b",
       "IPY_MODEL_0fa78a342be94571a268ee61f4e3c01d"
      ],
      "layout": "IPY_MODEL_2fa8386c7a7b47f4a5417002e5fec63a"
     }
    },
    "f86cfe1ddaa240ce8eaac38a82c20abd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ff749937033648f9a17b5f36a31d9765": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}