copa_robertaMultiple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "# import json\n",
    "# from torch.utils.data import Dataset, DataLoader\n",
    "# import transformers\n",
    "from transformers import RobertaModel, RobertaTokenizer, RobertaForMultipleChoice\n",
    "from torch import cuda\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>premise</th>\n",
       "      <th>choice1</th>\n",
       "      <th>choice2</th>\n",
       "      <th>question</th>\n",
       "      <th>label</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>My body cast a shadow over the grass.</td>\n",
       "      <td>The sun was rising.</td>\n",
       "      <td>The grass was cut.</td>\n",
       "      <td>cause</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The woman tolerated her friend's difficult beh...</td>\n",
       "      <td>The woman knew her friend was going through a ...</td>\n",
       "      <td>The woman felt that her friend took advantage ...</td>\n",
       "      <td>cause</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The women met for coffee.</td>\n",
       "      <td>The cafe reopened in a new location.</td>\n",
       "      <td>They wanted to catch up with each other.</td>\n",
       "      <td>cause</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The runner wore shorts.</td>\n",
       "      <td>The forecast predicted high temperatures.</td>\n",
       "      <td>She planned to run along the beach.</td>\n",
       "      <td>cause</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The guests of the party hid behind the couch.</td>\n",
       "      <td>It was a surprise party.</td>\n",
       "      <td>It was a birthday party.</td>\n",
       "      <td>cause</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The politician lost the election.</td>\n",
       "      <td>He ran negative campaign ads.</td>\n",
       "      <td>No one voted for him.</td>\n",
       "      <td>cause</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The stain came out of the shirt.</td>\n",
       "      <td>I patched the shirt.</td>\n",
       "      <td>I bleached the shirt.</td>\n",
       "      <td>cause</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The man got a discount on his groceries.</td>\n",
       "      <td>He greeted the cashier.</td>\n",
       "      <td>He used a coupon.</td>\n",
       "      <td>cause</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>The physician misdiagnosed the patient.</td>\n",
       "      <td>The patient filed a malpractice lawsuit agains...</td>\n",
       "      <td>The patient disclosed confidential information...</td>\n",
       "      <td>effect</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>The customer filed a complaint with the store ...</td>\n",
       "      <td>The sales associate undercharged the customer.</td>\n",
       "      <td>The sales associate acted rude to the customer.</td>\n",
       "      <td>cause</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             premise  \\\n",
       "0              My body cast a shadow over the grass.   \n",
       "1  The woman tolerated her friend's difficult beh...   \n",
       "2                          The women met for coffee.   \n",
       "3                            The runner wore shorts.   \n",
       "4      The guests of the party hid behind the couch.   \n",
       "5                  The politician lost the election.   \n",
       "6                   The stain came out of the shirt.   \n",
       "7           The man got a discount on his groceries.   \n",
       "8            The physician misdiagnosed the patient.   \n",
       "9  The customer filed a complaint with the store ...   \n",
       "\n",
       "                                             choice1  \\\n",
       "0                                The sun was rising.   \n",
       "1  The woman knew her friend was going through a ...   \n",
       "2               The cafe reopened in a new location.   \n",
       "3          The forecast predicted high temperatures.   \n",
       "4                           It was a surprise party.   \n",
       "5                      He ran negative campaign ads.   \n",
       "6                               I patched the shirt.   \n",
       "7                            He greeted the cashier.   \n",
       "8  The patient filed a malpractice lawsuit agains...   \n",
       "9     The sales associate undercharged the customer.   \n",
       "\n",
       "                                             choice2 question  label  idx  \n",
       "0                                 The grass was cut.    cause      0    0  \n",
       "1  The woman felt that her friend took advantage ...    cause      0    1  \n",
       "2           They wanted to catch up with each other.    cause      1    2  \n",
       "3                She planned to run along the beach.    cause      0    3  \n",
       "4                           It was a birthday party.    cause      0    4  \n",
       "5                              No one voted for him.    cause      1    5  \n",
       "6                              I bleached the shirt.    cause      1    6  \n",
       "7                                  He used a coupon.    cause      1    7  \n",
       "8  The patient disclosed confidential information...   effect      0    8  \n",
       "9    The sales associate acted rude to the customer.    cause      1    9  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "test_raw_data = pd.read_json('data/test.jsonl', lines=True)\n",
    "dev_raw_data = pd.read_json('data/dev.jsonl', lines=True)\n",
    "train_raw_data = pd.read_json('data/train.jsonl', lines=True)\n",
    "train_raw_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_sequence is:  {effect}The girl refused to eat her vegetables.\n",
      "{'input_ids': [0, 45152, 26715, 24303, 133, 1816, 3179, 7, 3529, 69, 8942, 4, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n",
      "['{', 'effect', '}', 'The', 'Ġgirl', 'Ġrefused', 'Ġto', 'Ġeat', 'Ġher', 'Ġvegetables', '.']\n",
      "test_sequence is:  {effect}The girl found a bug in her cereal.\n",
      "{'input_ids': [0, 45152, 26715, 24303, 133, 1816, 303, 10, 13673, 11, 69, 25629, 4, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n",
      "['{', 'effect', '}', 'The', 'Ġgirl', 'Ġfound', 'Ġa', 'Ġbug', 'Ġin', 'Ġher', 'Ġcereal', '.']\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "# test_sequence = \"{\" + \"effect\" + \"}\" + \"I ran the ice cube under warm water.\"\n",
    "test_sequence = \"{\"+ test_raw_data.iloc[28]['question'] + \"}\" + test_raw_data.iloc[28]['premise']\n",
    "print(\"test_sequence is: \", test_sequence)\n",
    "print(tokenizer(test_sequence))\n",
    "print(tokenizer.tokenize(test_sequence))\n",
    "# test 2\n",
    "test_sequence = \"{\"+ test_raw_data.iloc[1]['question'] + \"}\" + test_raw_data.iloc[1]['premise']\n",
    "print(\"test_sequence is: \", test_sequence)\n",
    "print(tokenizer(test_sequence))\n",
    "print(tokenizer.tokenize(test_sequence))\n",
    "\n",
    "print(test_raw_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(rawdata):\n",
    "    tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
    "    \n",
    "    # for i in range(0, rawdata.shape[0]):\n",
    "    for i in range(2, 5):\n",
    "        prompt = rawdata.iloc[i]['question'] + \".\" + rawdata.iloc[i]['premise']\n",
    "        choice0 = rawdata.iloc[i]['choice1']\n",
    "        choice1 = rawdata.iloc[i]['choice2']\n",
    "        label = torch.tensor(rawdata.iloc[i]['label'])\n",
    "        # label = torch.tensor(rawdata.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "\n",
    "        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True)\n",
    "        print(\"encoding['input_ids']: \", encoding['input_ids'])\n",
    "        print(\"encoding['input_ids'] with size of : \", encoding['input_ids'].size())\n",
    "        print(\"encoding['attention_mask']: \", encoding['attention_mask'])\n",
    "        print(\"label: \", label)\n",
    "    return encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "350\n",
      "50\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "print(train_raw_data.shape[0])\n",
    "print(dev_raw_data.shape[0])\n",
    "print(test_raw_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoding['input_ids']:  tensor([[    0, 27037,     4,   133,   390,  1145,    13,  3895,     4,     2,\n",
      "             2,   133, 16381, 14015,    11,    10,    92,  2259,     4,     2,\n",
      "             1],\n",
      "        [    0, 27037,     4,   133,   390,  1145,    13,  3895,     4,     2,\n",
      "             2,  1213,   770,     7,  2916,    62,    19,   349,    97,     4,\n",
      "             2]])\n",
      "encoding['input_ids'] with size of :  torch.Size([2, 21])\n",
      "encoding['attention_mask']:  tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "label:  tensor(1)\n",
      "encoding['input_ids']:  tensor([[    0, 27037,     4,   133,  7449,  5328, 13344,     4,     2,     2,\n",
      "           133,  1914,  6126,   239,  3971,     4,     2,     1,     1],\n",
      "        [    0, 27037,     4,   133,  7449,  5328, 13344,     4,     2,     2,\n",
      "          2515,  1904,     7,   422,   552,     5,  4105,     4,     2]])\n",
      "encoding['input_ids'] with size of :  torch.Size([2, 19])\n",
      "encoding['attention_mask']:  tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "label:  tensor(0)\n",
      "encoding['input_ids']:  tensor([[    0, 27037,     4,   133,  3958,     9,     5,   537, 20119,   639,\n",
      "             5, 16433,     4,     2,     2,   243,    21,    10,  2755,   537,\n",
      "             4,     2],\n",
      "        [    0, 27037,     4,   133,  3958,     9,     5,   537, 20119,   639,\n",
      "             5, 16433,     4,     2,     2,   243,    21,    10,  4115,   537,\n",
      "             4,     2]])\n",
      "encoding['input_ids'] with size of :  torch.Size([2, 22])\n",
      "encoding['attention_mask']:  tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "label:  tensor(0)\n"
     ]
    }
   ],
   "source": [
    "# tokenize data tests\n",
    "train_data = load_data(train_raw_data)\n",
    "# print(f'Training data loaded (length {len(train_data)})')\n",
    "# dev_data = load_data('data/dev.jsonl')\n",
    "# print(f'Dev data loaded (length {len(dev_data)})')\n",
    "# test_data = load_data('data/test.jsonl')\n",
    "# print(f'Test data loaded (length {len(test_data)})')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model_3, use only the very last hidden layer from Roberta.\n",
    "from torch import nn\n",
    "from transformers import RobertaConfig, RobertaModel\n",
    "\n",
    "class OurRobertaCOPA(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(OurRobertaCOPA, self).__init__()\n",
    "        # self.configuration = RobertaConfig()\n",
    "        # self.tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "        # self.l1 = RobertaModel(self.configuration)\n",
    "        self.l1 = RobertaModel.from_pretrained(\"roberta-base\")\n",
    "        self.l1.requires_grad = True\n",
    "        self.softmax = nn.Softmax(dim=0)\n",
    "        self.pre_classifier = torch.nn.Linear(768, 512)\n",
    "        self.dropout = torch.nn.Dropout(0.3)\n",
    "        # self.classifier = torch.nn.Linear(768, 5)\n",
    "        # hidden_dim=32 for later trials.\n",
    "        # self.lstm = nn.LSTM(768, 32, 1, bias=False)\n",
    "        self.output_layer = nn.Linear(512, 2)\n",
    "\n",
    "    def forward(self, sequence_1, sequence_2):\n",
    "        # Two input here\n",
    "        token_1 = tokenizer(sequence_1)\n",
    "        token_2 = tokenizer(sequence_2)\n",
    "        output_1 = self.l1(input_ids=torch.tensor(token_1[\"input_ids\"]).unsqueeze(0), attention_mask=torch.tensor(token_1[\"attention_mask\"]).unsqueeze(0))[0]\n",
    "        output_2 = self.l1(input_ids=torch.tensor(token_2[\"input_ids\"]).unsqueeze(0), attention_mask=torch.tensor(token_2[\"attention_mask\"]).unsqueeze(0))[0]\n",
    "        # RobertaModel(RobertaConfig())\n",
    "\n",
    "        # _, (hidden_rep_1, _) = self.lstm(output_1.unsqueeze(0))\n",
    "        # _, (hidden_rep_2, _) = self.lstm(output_2.unsqueeze(0))\n",
    "        # _, (hidden_rep_1, _) = self.lstm(output_1)\n",
    "        # _, (hidden_rep_2, _) = self.lstm(output_2)\n",
    "\n",
    "        hidden_rep_1 = torch.nn.ReLU()(self.pre_classifier(output_1[0])).squeeze(0)\n",
    "        hidden_rep_2 = torch.nn.ReLU()(self.pre_classifier(output_2[0])).squeeze(0)\n",
    "        pooler_1 = hidden_rep_1[:, 0]\n",
    "        pooler_2 = hidden_rep_2[:, 0]\n",
    "        # hidden_rep_1 = self.pre_classifier(output_1[0]).squeeze(0)\n",
    "        # hidden_rep_2 = self.pre_classifier(output_2[0]).squeeze(0)\n",
    "        # print(\"-------hidden_rep_1:\")\n",
    "        # print(hidden_rep_1)\n",
    "        # print(hidden_rep_1.size())\n",
    "        # print(\"-------hidden_rep_2:\")\n",
    "        # print(hidden_rep_2)\n",
    "        # print(hidden_rep_2.size())\n",
    "        \n",
    "        # hidden_rep = torch.cat((hidden_rep_1.unsqueeze(1), hidden_rep_2.unsqueeze(1)), 1)\n",
    "        # hidden_rep = self.dropout(torch.cat((hidden_rep_1, hidden_rep_2), 0))\n",
    "        hidden_rep = self.dropout(torch.cat((pooler_1, pooler_2), 0))\n",
    "\n",
    "        print(\"-------hidden_rep:\")\n",
    "        # print(hidden_rep)\n",
    "        print(hidden_rep.size())\n",
    "\n",
    "        output = self.output_layer(hidden_rep.unsqueeze(0))\n",
    "        print(\"-------output:\")\n",
    "        # print(output)\n",
    "        print(output.size())\n",
    "        print(\"--------------\")\n",
    "\n",
    "        output_squezzed = output.squeeze(0).squeeze(0)\n",
    "        print(\"-------output_squezzed:\")\n",
    "        print(output_squezzed)\n",
    "        print(output_squezzed.size())\n",
    "        print(\"--------------\")\n",
    "        \n",
    "        # y_hat = softmax(output_squezzed)\n",
    "        # y_sum =  torch.sum(y_hat, 0)\n",
    "        # col1= torch.sum(y_hat, 0)[0]\n",
    "        # col2 = torch.sum(y_hat, 0)[1]\n",
    "        # y_result = torch.tensor(torch.argmax(y_sum)).type(torch.FloatTensor)\n",
    "        # y_result = torch.tensor(y_sum)\n",
    "        \n",
    "        return output_squezzed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForMultipleChoice: ['lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.bias']\n",
      "- This IS expected if you are initializing RobertaForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForMultipleChoice were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaForMultipleChoice(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): RobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): RobertaPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialization\n",
    "tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
    "# model = OurRobertaCOPA()\n",
    "model = RobertaForMultipleChoice.from_pretrained('roberta-base')\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------Epoch: 1--------------\n",
      "Training for epoch 1.......\n",
      "train_loss:  tensor(4.3557, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[-7.3646, -3.0218]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(0.0139, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[ 3.3072, -0.9613]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_10868/1522285655.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     33\u001b[0m         \u001b[0mencoding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mprompt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprompt\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mchoice0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchoice1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'pt'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m         \u001b[1;31m# outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 35\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     36\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m         \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, token_type_ids, attention_mask, labels, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m   1305\u001b[0m         )\n\u001b[0;32m   1306\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1307\u001b[1;33m         outputs = self.roberta(\n\u001b[0m\u001b[0;32m   1308\u001b[0m             \u001b[0mflat_input_ids\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1309\u001b[0m             \u001b[0mposition_ids\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mflat_position_ids\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    848\u001b[0m             \u001b[0mpast_key_values_length\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpast_key_values_length\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    849\u001b[0m         )\n\u001b[1;32m--> 850\u001b[1;33m         encoder_outputs = self.encoder(\n\u001b[0m\u001b[0;32m    851\u001b[0m             \u001b[0membedding_output\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    852\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mextended_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    522\u001b[0m                 )\n\u001b[0;32m    523\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 524\u001b[1;33m                 layer_outputs = layer_module(\n\u001b[0m\u001b[0;32m    525\u001b[0m                     \u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    526\u001b[0m                     \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    408\u001b[0m         \u001b[1;31m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    409\u001b[0m         \u001b[0mself_attn_past_key_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mpast_key_value\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 410\u001b[1;33m         self_attention_outputs = self.attention(\n\u001b[0m\u001b[0;32m    411\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    412\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    335\u001b[0m         \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    336\u001b[0m     ):\n\u001b[1;32m--> 337\u001b[1;33m         self_outputs = self.self(\n\u001b[0m\u001b[0;32m    338\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    339\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m   1101\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m         \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\light\\Documents\\RoadToCreation\\python\\CUDA11\\lib\\site-packages\\transformers\\models\\roberta\\modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    237\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m         \u001b[1;31m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 239\u001b[1;33m         \u001b[0mattention_scores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    240\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key\"\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key_query\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "ce = nn.CrossEntropyLoss()\n",
    "softmax = nn.Softmax(dim=0)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "\n",
    "epochs = 10\n",
    "per_num_epoch = 1\n",
    "\n",
    "# train_acc = np.zeros(epochs)\n",
    "train_loss_by_epoch = np.zeros(epochs)\n",
    "dev_acc = np.zeros(epochs)\n",
    "dev_loss_by_epoch = np.zeros(epochs)\n",
    "\n",
    "start_time = datetime.now()\n",
    "\n",
    "for j in range(epochs):\n",
    "    if j % per_num_epoch == 0:\n",
    "        print('--------------Epoch: ' + str(j+1) + '--------------')\n",
    "    \n",
    "    if j % per_num_epoch == 0:\n",
    "        print(f'Training for epoch {j + 1}.......')\n",
    "    \n",
    "    av_train_loss = 0\n",
    "    # print(\"av_train_loss_original: \", av_train_loss)\n",
    "    model.train()\n",
    "    for i in range(0, train_raw_data.shape[0]):\n",
    "        # print(\"av_train_loss_track: \", av_train_loss)\n",
    "        prompt = train_raw_data.iloc[i]['question'] + \". \" + train_raw_data.iloc[i]['premise']\n",
    "        choice0 = train_raw_data.iloc[i]['choice1']\n",
    "        choice1 = train_raw_data.iloc[i]['choice2']\n",
    "        label = torch.tensor(train_raw_data.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "        # print(\"label is: \", label)\n",
    "\n",
    "        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)\n",
    "        # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)\n",
    "        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=label)\n",
    "        \n",
    "        train_loss = outputs.loss\n",
    "        train_logits = outputs.logits\n",
    "        av_train_loss += train_loss\n",
    "\n",
    "        if i == 0:\n",
    "            print(\"train_loss: \", train_loss)\n",
    "            print(\"train_logits: \", train_logits)\n",
    "            print(\"label: \", label)\n",
    "        if i == 1:\n",
    "            print(\"train_loss: \", train_loss)\n",
    "            print(\"train_logits: \", train_logits)\n",
    "            print(\"label: \", label)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        train_loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        # learning rate decay\n",
    "        # if j == 25:\n",
    "        #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
    "        # elif j == 50:\n",
    "        #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-7)\n",
    "\n",
    "    train_loss_by_epoch[j] = av_train_loss / train_raw_data.shape[0]\n",
    "    print(\"av_train_loss: \", train_loss_by_epoch[j])\n",
    "\n",
    "    # validation\n",
    "    # if (j + 1) % per_num_epoch == 0:\n",
    "    #     print(f'.......Validating for epoch {j + 1}')\n",
    "    if (j) % per_num_epoch == 0:\n",
    "        print(f'.......Validating for epoch {j + 1}')\n",
    "        av_dev_loss = 0\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            for i in range(0, dev_raw_data.shape[0]):\n",
    "                # print(\"av_dev_loss_track: \", av_dev_loss)\n",
    "                prompt = dev_raw_data.iloc[i]['question'] + \". \" + dev_raw_data.iloc[i]['premise']\n",
    "                choice0 = dev_raw_data.iloc[i]['choice1']\n",
    "                choice1 = dev_raw_data.iloc[i]['choice2']\n",
    "                label = torch.tensor(dev_raw_data.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "\n",
    "                encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)\n",
    "                # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)\n",
    "                outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=label)\n",
    "                \n",
    "                dev_loss = outputs.loss\n",
    "                dev_logits = outputs.logits\n",
    "                av_dev_loss += dev_loss\n",
    "                \n",
    "                if i == 0:\n",
    "                    print(\"dev_loss: \", dev_loss)\n",
    "                    print(\"dev_logits: \", dev_logits)\n",
    "                    print(\"label: \", label)\n",
    "                if i == 1:\n",
    "                    print(\"dev_loss: \", dev_loss)\n",
    "                    print(\"dev_logits: \", dev_logits)\n",
    "                    print(\"label: \", label)\n",
    "\n",
    "                #calculate accuracy\n",
    "                y_pred = 1 if outputs.logits[0][1] > outputs.logits[0][0] else 0\n",
    "                y_pred = torch.tensor(y_pred).unsqueeze(0).to(device)\n",
    "                # print(\"y_pred: \", y_pred)\n",
    "                # print(\"label: \", label)\n",
    "                # print(\"y_pred =? label: \", y_pred == label)\n",
    "                if y_pred == label:\n",
    "                    dev_acc[j] += 1\n",
    "                # print(\"dev_acc[j]: \", dev_acc[j])\n",
    "\n",
    "        dev_acc[j] /= dev_raw_data.shape[0]\n",
    "        dev_loss_by_epoch[j] = av_dev_loss / dev_raw_data.shape[0]\n",
    "\n",
    "end_time = datetime.now()\n",
    "print(f'Training completed in {str(end_time - start_time)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA00AAAG5CAYAAABSn98KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABzBElEQVR4nO3dd3hUZcL+8e+TXkgDQnpI6ARIkA6xYwEBUURfcW24u/7ebe/u6tp37b1ued3i7oquq1gAFbBgbxRpktB7SEIoISGd9Of3R8Z9o2IIkMmZzNyf65rrSk6bezQJc895znOMtRYRERERERE5Oj+nA4iIiIiIiHgylSYREREREZE2qDSJiIiIiIi0QaVJRERERESkDSpNIiIiIiIibVBpEhERERERaYNKk4iIiMOMMWnGGGuMCXA6i4iIfJdKk4iIdBhjTJ4x5pxOfs5bjTGfHWV5T2NMvTFmqDEmyBjzhDGm0BhT5cr5+zaOaY0x1a5tv37c7NYXIiIiHkufaImISFf3b+B+Y0y6tXZ3q+WXA+uttRuMMXcBo4AxwD6gN3D6MY6bZa3d4ZbEIiLSpehMk4iIuJ0xJtgY83tjTJHr8XtjTLBrXU9jzGJjTJkxptQY87kxxs+17hZjzF5jTKUxZqsxZuK3j22tLQQ+Aq761qqrgX+5vh4NvG6tLbIt8qy1/+IEGGPuNsbMM8a84sq11hiT1Wr9YGPMJ67Xs9EYc2GrdaGuM157jDHlxpgvjDGhrQ7/A2NMvjHmkDHmjhPJJyIiHU+lSUREOsMdwDhgOJBFyxmf37rW3QgUArFAHHA7YI0xA4GfA6OttRHA+UDe9xz/eVqVJte+w4GXXItWADcYY35qjBlmjDEn+XqmA68B3V3P8YYxJtAYEwgsAt4DegG/AF505QF4HBgJTHDtezPQ3Oq4pwIDgYnAncaYwSeZU0REOoBKk4iIdIYfAPdaaw9aa4uBe/i/ktMAJAC9rbUN1trPrbUWaAKCgQxjTKDr7NDO7zn+60CcMWaC6/urgXdczwXwEPCIK8dqYK8x5ppjZF7rOlv09eP8VuvWWGvnWWsbgCeBEFpK4TigG/CwtbbeWvsRsBiY5Tp7dh3wS2vtXmttk7V2mbW2rtVx77HWHrHW5gA5tBRMERFxmEqTiIh0hkRgT6vv97iWATwG7ADeM8bsMsbcCuC6nuhXwN3AQWPMy8aYRI7CWltDy5mfq11nkX7A/w3Nw1VQnrbWZgPRwAPAs8c4kzPCWhvd6rGk1bqCVsdupuVMWaLrUeBa1vq1JgE9aSlX31f8APa3+rqGlgImIiIOU2kSEZHOUETL5AtfS3Utw1pbaa290VrbB7iQlmF0E13rXrLWnura19Jytuj7PA9cBpwLRNAyTO47XGdyngYOAxkn+HpSvv7CdQYp2fV6ioCUr6/JckkF9gKHgFqg7wk+p4iIOESlSUREOlqgMSak1SMAmAv81hgTa4zpCdxJy6x3GGOmGmP6uc4QldMyLK/ZGDPQGHO2a8KIWuAI37z+59s+B8qAZ4CXrbX1X68wxvzKGHOmayKGANfQvAjgqxN8jSONMTNcr+1XQB0t1019ScsZoptd1zidCUxz5WkGngWeNMYkGmP8jTHjv54QQ0REPJdKk4iIdLS3aSk4Xz/uBu6n5VqiXGA9sNa1DKA/8AFQBSwH/myt/ZiW65kepuUMzX5aJla47fue1HUd1L9oOSv17ZnxaoAnXMc5BPwMuMRau6uN15Hzrfs0/b7VujeB/6LlbNVVwAzX9Vj1tJSkya7n+TNwtbV2i2u/37he/yqglJYzZ/q3WETEw5mWf2NERESkPYwxdwP9rLVXOp1FREQ6hz7dEhERERERaYNKk4iIiIiISBs0PE9ERERERKQNOtMkIiIiIiLShgCnA3SGnj172rS0NKdjiIiIiIiIh1qzZs0ha23s0db5RGlKS0tj9erVTscQEREREREPZYzZ833rNDxPRERERESkDSpNIiIiIiIibVBpEhERERERaYNPXNMkIiIiItKVNTQ0UFhYSG1trdNRuryQkBCSk5MJDAxs9z4qTSIiIiIiHq6wsJCIiAjS0tIwxjgdp8uy1lJSUkJhYSHp6ent3k/D80REREREPFxtbS09evRQYTpJxhh69Ohx3GfsVJpERERERLoAFaaOcSL/HVWaRERERERE2qDSJCIiIiIi0gaVJhERERERaVNZWRl//vOfj3u/Cy64gLKysuPe79prr2XevHnHvZ+7qDSJiIiIiEibvq80NTY2trnf22+/TXR0tJtSdR5NOS4iIiIi0oXcs2gjm4oqOvSYGYmR3DVtyPeuv/XWW9m5cyfDhw8nMDCQkJAQYmJi2LJlC9u2beOiiy6ioKCA2tpafvnLX3L99dcDkJaWxurVq6mqqmLy5MmceuqpLFu2jKSkJN58801CQ0OPme3DDz/kN7/5DY2NjYwePZq//OUvBAcHc+utt7Jw4UICAgI477zzePzxx3nttde455578Pf3Jyoqis8++6xD/vuoNImIiIiISJsefvhhNmzYwLp16/jkk0+YMmUKGzZs+M+9jp599lm6d+/OkSNHGD16NJdccgk9evT4xjG2b9/O3Llz+fvf/85ll13G/PnzufLKK9t83traWq699lo+/PBDBgwYwNVXX81f/vIXrrrqKl5//XW2bNmCMeY/QwDvvfdelixZQlJS0gkNC/w+Kk0iIiIiIl1IW2eEOsuYMWO+cXPYP/7xj7z++usAFBQUsH379u+UpvT0dIYPHw7AyJEjycvLO+bzbN26lfT0dAYMGADANddcw9NPP83Pf/5zQkJC+OEPf8jUqVOZOnUqANnZ2Vx77bVcdtllzJgxowNeaQtd0yQiIiJeoaquEWut0zFEfEJ4ePh/vv7kk0/44IMPWL58OTk5OZxyyilHvXlscHDwf7729/c/5vVQbQkICGDlypXMnDmTxYsXM2nSJAD++te/cv/991NQUMDIkSMpKSk54edoTaVJREREurzK2gbOfOxjbnwtx+koIl4pIiKCysrKo64rLy8nJiaGsLAwtmzZwooVKzrseQcOHEheXh47duwA4IUXXuCMM86gqqqK8vJyLrjgAp566ilyclp+93fu3MnYsWO59957iY2NpaCgoENyaHieiIiIdHmvrS7kUFU9C9buZeKgOKZkJjgdScSr9OjRg+zsbIYOHUpoaChxcXH/WTdp0iT++te/MnjwYAYOHMi4ceM67HlDQkKYM2cOl1566X8mgvjv//5vSktLmT59OrW1tVhrefLJJwG46aab2L59O9ZaJk6cSFZWVofkML5wGnvUqFF29erVTscQERERN2hqtpz9xCd0Dw+i2UJ+STVLfnU6vSJDnI4m0mE2b97M4MGDnY7hNY7239MYs8ZaO+po22t4noiIiHRpH285yJ6SGn54ajpPXJpFTX0Tty5Yr+ubRKTDuLU0GWMmGWO2GmN2GGNuPcr6p4wx61yPbcaYslbr3jXGlBljFn9rnxddx9xgjHnWGBPoztcgIiIinm3Ost0kRIVw/pB4+vXqxq2TB/HRloO8sqpjrmUQEff52c9+xvDhw7/xmDNnjtOxvsNt1zQZY/yBp4FzgUJglTFmobV209fbWGt/3Wr7XwCntDrEY0AY8P++degXga8ndH8J+BHwlw5/ASIiIuLxtu6vZOmOEm6eNJBA/5bPgq8Zn8b7mw5w3+JNZPfrSUr3MIdTinQMay3GGKdjdKinn36605/zRM5Cu/NM0xhgh7V2l7W2HngZmN7G9rOAuV9/Y639EPjOFB3W2retC7ASSO7Y2CIiItJVPLdsNyGBfswanfqfZX5+hscuzcLPGG58NYemZg3Tk64vJCSEkpISDTs9SdZaSkpKCAk5vmse3Tl7XhLQ+rx4ITD2aBsaY3oD6cBH7T24a1jeVcAvv2f99cD1AKmpqUfbRERERLqww9Uts+XNGJFETHjQN9YlRYdy14VD+M1rOTz7xW5+fHofh1KKdIzk5GQKCwspLi52OkqXFxISQnLy8Z138ZQpxy8H5llrm45jnz8Dn1lrPz/aSmvtM8Az0DJ73slHFBEREU8yd1U+dY3NXDsh/ajrLxmRxHsb9/PYe1s5Y2AsA+IiOjmhSMcJDAwkPf3oP+vifu4cnrcXSGn1fbJr2dFcTquhecdijLkLiAVuOOF0IiIi0mU1NDXzwvI9ZPfrwcD4o5chYwwPzhhGRHAAv35lHfWNzZ2cUkS8hTtL0yqgvzEm3RgTREsxWvjtjYwxg4AYYHl7DmqM+RFwPjDLWqu/fiIiIj5oycb97CuvZfb3nGX6Ws9uwTw4Yxgbiyr434+2d1I6EfE2bitN1tpG4OfAEmAz8Kq1dqMx5l5jzIWtNr0ceNl+66o2Y8znwGvARGNMoTHmfNeqvwJxwHLXVOV3uus1iIiIiGeaszSP3j3COHtQr2Nue/6QeC4ZkczTn+xkXUGZ+8OJiNcxvjADx6hRo+zq1audjiEiIiIdILewjAv/dyl3Ts3gulPbd41HRW0Dk576jJAgf976xWmEBvm7OaWIdDXGmDXW2lFHW+fWm9uKiIiIdLQ5S/PoFhzApaPaP/tVZEggj1+axa7iah55d4sb04mIN1JpEhERkS7jYEUti3OLmDkymYiQwOPad0K/nlw7IY3nluWxdMchNyUUEW+k0iQiIiJdxr+/zKex2XLthLQT2v+WSYPoExvOTa/lUFHb0LHhRMRrqTSJiIhIl1DX2MRLX+7h7IG9SOsZfkLHCA3y58nLhnOgso57Fm7q4IQi4q1UmkRERKRLWJSzj0NV9czOPrkbfA5PieZnZ/Zl/tpC3t2wv4PSiYg3U2kSERERj2etZc7S3QyI60Z2vx4nfbyfn92foUmR3PH6eg5V1XVAQhHxZipNIiIi4vFW5R1mY1EF105Ixxhz0scLCvDjycuGU1nXyG0L1uMLt2ARkROn0iQiIiIeb87S3USHBXLxKUkddswBcRHcfP5A3t90gPlr93bYcUXE+6g0iYiIiEcrPFzDko37uXx0aofflPa67HTGpHfnnoUbKTxc06HHFhHvodIkIiIiHu2F5XswxnD1+N4dfmw/P8MTl2bRbC03vZZLc7OG6YnId6k0iYiIiMeqqW9k7sp8Jg2JJzE61C3PkdI9jDunZbB8VwnPL89zy3OISNem0iQiIiIea8HavVTUNjI7O82tz3PZqBTOHtSLh9/Zwo6DVW59LhHpelSaRERExCNZa3luWR7DkqIY2TvGrc9ljOHhS4YRFuTPja+uo7Gp2a3PJyJdi0qTiIiIeKTPtx9ix8EqZmendcg048fSKyKEBy4eRk5hOX/+ZKfbn09Eug6VJhEREfFIc5bupme3YKZkJnTac14wLIGLhifyxw+3s76wvNOeV0Q8m0qTiIiIeJxdxVV8vLWYK8elEhzQsdOMH8s9Fw6lZ7dgfv3qOmobmjr1uUXEM6k0iYiIiMd5flkeQf5+/GBsx08zfixRYYE8OjOTHQereHzJ1k5/fhHxPCpNIiIi4lEqahuYt6aQqVkJxEYEO5Lh9AGxXDWuN/9cupsVu0ocySAinkOlSURERDzKq6sKqK5v4rrsdEdz3HbBIHp3D+M3r+VQWdvgaBYRcZZKk4iIiHiMpmbL88vzGJ0Ww9CkKEezhAUF8MRlWRSVHeH+xZsdzSIizlJpEhEREY/x4eYDFJQeYbbDZ5m+NrJ3d/77jL68srqADzYdcDqOiDhEpUlEREQ8xpyleSRFh3JeRpzTUf7jV+cMYHBCJLcuWE9pdb3TcUTEASpNIiIi4hE276tg+a4SrhrfmwB/z3mLEhTgx5OXZVF+pJ47Xl+PtdbpSCLSyTznL5KIiIj4tOeW5hES6Mflo1OcjvIdgxMiueHcgbyzYT9vrityOo6IdDKVJhEREXFcaXU9b6zby4wRyUSHBTkd56iuP70PI3vHcOebG9hXfsTpOCLSiVSaRERExHFzV+ZT19jM7AlpTkf5Xv5+hicuzaKhyXLzvFwN0xPxISpNIiIi4qiGpmZeWL6H0/r3pH9chNNx2pTWM5w7pgzm8+2H+PeKPU7HEZFOotIkIiIijnpnw372V9QyOzvN6Sjt8oOxqZw+IJYH3t7M7kPVTscRkU6g0iQiIiKOmrN0N+k9wzlzQC+no7SLMYZHL8kkOMCfG19dR2NTs9ORRMTNVJpERETEMesKyvgqv4xrxvfGz884Hafd4qNCuHf6ENbml/G3z3Y5HUdE3EylSURERBwzZ+luIoIDmDnK86YZP5YLsxKZkpnA7z/YxqaiCqfjiIgbqTSJiIiIIw5U1PJW7j4uHZVCt+AAp+McN2MM908fSnRYEDe8uo66xianI4mIm6g0iYiIiCP+vWIPTdZyrQdPM34sMeFBPHpJJlv2V/LU+9udjiMibqLSJCIiIp2utqGJl77MZ+KgOFJ7hDkd56ScNagXs8ak8LfPdrIqr9TpOCLiBipNIiIi0ukW5hRRUl3PdV1kmvFjuWNKBskxodz4ag7VdY1OxxGRDqbSJCIiIp3KWsucpXkMjItgfN8eTsfpEN2CA3ji0uEUHK7hwbc3Ox1HRDqYW0uTMWaSMWarMWaHMebWo6x/yhizzvXYZowpa7XuXWNMmTFm8bf2STfGfOk65ivGmCB3vgYRERHpWF/uLmXzvgpmZ6dhTNeZZvxYxqR358en9eHFL/P5eOtBp+OISAdyW2kyxvgDTwOTgQxgljEmo/U21tpfW2uHW2uHA38CFrRa/Rhw1VEO/QjwlLW2H3AY+KEb4ouIiIibzFm6m5iwQC46JcnpKB3uhnMHMCCuG7fMy6Wspt7pOCLSQdx5pmkMsMNau8taWw+8DExvY/tZwNyvv7HWfghUtt7AtHwcdTYwz7XoeeCiDswsIiIiblRQWsN7mw4wa0wqIYH+TsfpcCGB/jx52XBKq+u5882NTscRkQ7iztKUBBS0+r7Qtew7jDG9gXTgo2McswdQZq39+grLto55vTFmtTFmdXFx8XEFFxEREfd4flkefsZw1fjeTkdxm6FJUfxyYn8W5hSxKKfI6Tgi0gE8ZSKIy4F51toOuyuctfYZa+0oa+2o2NjYjjqsiIiInKDqukZeWV3A5KHxJESFOh3HrX5yZl+yUqL53ZsbOFBR63QcETlJ7ixNe4GUVt8nu5YdzeW0GprXhhIg2hjz9W3D2zqmiIiIeJD5awuprG1kdna601HcLsDfjycvy6K2oYlb5udirXU6koicBHeWplVAf9dsd0G0FKOF397IGDMIiAGWH+uAtuUvzsfATNeia4A3OyyxiIiIuEVzs+W5pXlkJUcxIjXa6Tidom9sN26bPJhPthbz8qqCY+8gIh7LbaXJdd3Rz4ElwGbgVWvtRmPMvcaYC1ttejnwsv3WRzDGmM+B14CJxphCY8z5rlW3ADcYY3bQco3TP931GkRERKRjfLq9mF2Hqpmdne5V04wfy1XjepPdrwf3Ld5EfkmN03FE5AQZXzhdPGrUKLt69WqnY4iIiPisq59dyZZ9FXxxy9kEBXjKJdWdo6jsCOf//jMGxUfw8vXj8ffzndIo0pUYY9ZYa0cdbZ1v/dUSERGRTrfjYBWfbSvmqnG9fa4wASRGh3LPhUNYlXeYf3y+y+k4InICfO8vl4iIiHSq55btJijAjyvGpjodxTEXn5LE+UPieOK9bWzdX3nsHUTEo6g0iYiIiNuU1zQwf81epmcl0qNbsNNxHGOM4cGLhxEZGsCvX1lHfWOz05FE5DioNImIiIjbvLI6nyMNTT4xzfix9OgWzIMXD2PTvgr++OF2p+OIyHFQaRIRERG3aGxq5vllexib3p2MxEin43iE84bEc+nIZP78yQ7W5h92Oo6ItJNKk4iIiLjFB5sPsLfsiM4yfcud0zJIiArlxldzOFLf5HQcEWkHlSYRERFxi2eX5pEcE8q5GXFOR/EoESGBPHZpJrsPVfPwO5udjiMi7aDSJCIiIh1uY1E5K3eXcs34NN2X6Cgm9O3JddnpPL98D19sP+R0HBE5BpUmERER6XBzluYRFuTPZaNTnI7isW6eNJC+seHcNC+H8iMNTscRkTaoNImIiEiHOlRVx8J1RVwyIpmo0ECn43iskEB/nrxsOAcr67hn4Uan44hIG1SaREREpEO99GU+9U3NXJud5nQUj5eVEs3Pz+rHgq/28u6GfU7HEZHvodIkIiIiHaa+sZkXVuzhjAGx9I3t5nScLuHnZ/djWFIUt7++geLKOqfjiMhRqDSJiIhIh3l7/T6KK+uYrbNM7Rbo78eTl2VRVdfIbQtysdY6HUlEvkWlSURERDqEtZY5S3fTJzac0/vHOh2nS+kfF8HN5w/kg80HeW1NodNxRORbVJpERESkQ6zNLyOnsJzZE9Lw0zTjx+267HTGpnfn3kWbKCitcTqOiLSi0iQiIiIdYs7S3USEBDBjRLLTUbokPz/D45dmAXDTvByamzVMT8RTqDSJiIjISdtXfoR3Nuzn8tEphAcHOB2ny0rpHsadUzNYsauUOcvynI4jIi4qTSIiInLSXli+B2stV49PczpKl3fpqGTOGdyLR97dwo6DlU7HERFUmkREROQk1TY0MXdlPudmxJHSPczpOF2eMYaHZmTSLTiAG17NoaGp2elIIj5PpUlEREROyhtf7eVwTQOzs9OdjuI1YiOCeeCioeQWlvP0xzucjiPi81SaRERE5IS1TDOex+CESMamd3c6jleZPCyBi09J4k8f7SC3sMzpOCI+TaVJRERETtjynSVsPVDJ7Ow0jNE04x3t7guHENstmF+/so7ahian44j4LJUmEREROWHPLs2jR3gQF2YlOh3FK0WFBvLYpZnsLK7msSVbnY4j4rNUmkREROSE7Cmp5sMtB7hibCohgf5Ox/Fap/WP5erxvfnnF7tZtvOQ03FEfJJKk4iIiJyQ55ftwd8YrhzX2+koXu/WyYNI7xnOTa/lUlnb4HQcEZ+j0iQiIiLHraqukddWFzAlM4G4yBCn43i9sKAAnrgsi33lR7h30San44j4HJUmEREROW7zVhdQWdeoacY70YjUGH5yZl9eW1PI+5sOOB1HxKeoNImIiMhxaW62PL98D6ekRjM8JdrpOD7llxMHMDghktsW5FJSVed0HBGfodIkIiIix+WTbQfZfahaZ5kcEBTgx1P/lUXFkUbueH0D1lqnI4n4BJUmEREROS5zluYRHxnC5KHxTkfxSYPiI7nhvAG8u3E/b6zb63QcEZ+g0iQiIiLttv1AJZ9vP8RV43sT6K+3EU758Wl9GNU7hjvf3EhR2RGn44h4Pf21ExERkXabsyyP4AA/Zo1JdTqKT/P3MzxxWRZNzZab5uXQ3KxheiLupNIkIiIi7VJWU8+CtYVcNDyJ7uFBTsfxeb17hPPbKRks3VHCCyv2OB1HxKupNImIiEi7vLyqgNqGZmafmuZ0FHGZNSaFMwfG8tA7m9lVXOV0HBGvpdIkIiIix9TY1My/luUxvk8PBsVHOh1HXIwxPHJJJsEB/tzwag6NTc1ORxLxSipNIiIickzvbTpAUXkts7PTnI4i3xIXGcL9Fw1lXUEZf/10p9NxRLySW0uTMWaSMWarMWaHMebWo6x/yhizzvXYZowpa7XuGmPMdtfjmlbLZxlj1htjco0x7xpjerrzNYiIiAjMWbqblO6hTBwc53QUOYppWYlMzUzg9x9sZ8PecqfjiHgdt5UmY4w/8DQwGcgAZhljMlpvY639tbV2uLV2OPAnYIFr3+7AXcBYYAxwlzEmxhgTAPwBOMtamwnkAj9312sQERER2LC3nFV5h7lmfBr+fsbpOPI97ps+lO7hQdz4ag61DU1OxxHxKu480zQG2GGt3WWtrQdeBqa3sf0sYK7r6/OB9621pdbaw8D7wCTAuB7hxhgDRAJF7noBIiIiAs8u3U14kD+XjU5xOoq0ISY8iEdmZrL1QCVPvb/N6TgiXsWdpSkJKGj1faFr2XcYY3oD6cBHbe1rrW0AfgKsp6UsZQD//J5jXm+MWW2MWV1cXHwyr0NERMRnFVfWsThnHzNHJhMZEuh0HDmGswb24oqxqTzz+S5W7i51Oo6I1/CUiSAuB+ZZa9s8l2yMCaSlNJ0CJNIyPO+2o21rrX3GWjvKWjsqNja2o/OKiIj4hBe/3EN9UzPXTEhzOoq00x0XDCYlJowbX1tHVV2j03FEvII7S9NeoPV5/GTXsqO5nP8bmtfWvsMBrLU7rbUWeBWY0EF5RUREpJW6xib+vSKfswbG0ie2m9NxpJ3CgwN44rIsCg8f4YG3NjsdR8QruLM0rQL6G2PSjTFBtBSjhd/eyBgzCIgBlrdavAQ4zzX5QwxwnmvZXiDDGPP1qaNzAf01EBERcYO3cvdxqKqO2dnpTkeR4zQ6rTvXn96HuSvz+XjLQafjiHR5bitN1tpGWma2W0JLsXnVWrvRGHOvMebCVpteDrzsOnP09b6lwH20FK9VwL2uSSGKgHuAz4wxubSceXrQXa9BRETEV1lrmbM0j369unFaf93doyu64dwBDIyL4Ob5uRyurnc6jkiXZlp1Fa81atQou3r1aqdjiIiIdBmr80qZ+dfl3H/RUK4c19vpOHKCNhaVc9HTSzl/SDz/e8UIp+OIeDRjzBpr7aijrfOUiSBERETEg8xZmkdkSAAzRhx14lvpIoYkRvGrcwawOHcfC3N0lxaRE6XSJCIiIt9QVHaEdzfuZ9aYVMKCApyOIyfp/53eh1NSo/ndGxs4UFHrdByRLkmlSURERL7hX8v3YK3lqvEalucNAvz9ePKy4dQ1NnHzvFx84dIMkY6m0iQiIiL/caS+ibkr8zl/SDzJMWFOx5EOkt4znNsvGMyn24p5aWW+03FEuhyVJhEREfmP17/aS/mRBk0z7oWuHNub0/r35P7Fm8k7VO10HJEuRaVJREREgJZpxp9btpshiZGMTotxOo50MD8/w6MzMwnwN/zmtRyamjVMT6S9VJpEREQEgKU7Sth2oIrZ2ekYY5yOI26QEBXKvdOHsHrPYf7++S6n44h0GSpNIiIiAsCcpbvp2S2IaVkJTkcRN7poeBKTh8bz5Hvb2Lyvwuk4Il2CSpOIiIiQd6iaj7Ye5IqxvQkO8Hc6jriRMYb7LxpKZGggN7yaQ31js9ORRDyeSpOIiIjw3LI8AvwMV45LdTqKdIIe3YJ5eMYwNu+r4A8fbnM6jojHU2kSERHxcZW1DcxbU8jUzER6RYQ4HUc6yTkZcVw2Kpm/fLKTNXsOOx1HxKOpNImIiPi411YXUlXXyHWaZtzn/G5qBglRodz46jpq6hudjiPisVSaREREfFhTs+W5ZXmM6h3DsOQop+NIJ4sICeSJy7LIK6nhuWV5TscR8VgqTSIiIj7soy0HyS+t0c1sfdi4Pj0Y2TuGheuKnI4i4rFUmkRERHzYnKW7SYwK4fwhcU5HEQdNzUxgy/5KdhysdDqKiEdSaRIREfFRW/ZXsGxnCVeNTyPAX28JfNmUYQkYA4ty9jkdRcQj6S+kiIiIj3puaR4hgX7MGpPidBRxWK/IEMamd2dRbhHWWqfjiHgclSYRke9ReLhGQ1XEa5VW1/P6V3u5+JRkosOCnI4jHmBaViK7iqvZvE9/90S+TaVJROQoKmobuPSvy7nsbyuobWhyOo5Ih5u7Mp+6xmZmZ6c5HUU8xOShCfj7GRblakIIkW9TaRIROYqH3t7CvvJaSqvrNaOUeJ2GpmZeWL6HU/v1ZEBchNNxxEN0Dw8iu19PFmuInsh3qDSJiHzL0h2HmLsyn+tP78Og+AieXbpbbyDEq7y7YT/7K2p1lkm+Y2pmAgWlR8gpLHc6iohHUWkSEWmluq6RWxfkkt4znBvOHcDs7DS27K9k+a4Sp6OJdJg5S3eT1iOMswb2cjqKeJjzh8QT6G9YnKMz7CKtqTSJiLTy2JKtFB4+wqMzMwkJ9Gf68CS6hwcxZ2me09FEOkROQRlr88u4ZkIafn7G6TjiYaJCAzljQCyLc/fR3Kwz7CJfU2kSEXFZlVfK88vzuGZ8GqPTugMQEujPFWNS+WDzAfaUVDucUOTkzVm6m27BAcwcmex0FPFQ07IS2V9Ry5r8w05HEfEYKk0iIkBtQxO3zMslOSaUm84f+I11V43vjb8xPLcsz5lwIh3kYEUtb63fx6WjkokICXQ6jnioiYPjCA7w0xA9kVZUmkREgKfe38auQ9U8PCOT8OCAb6yLiwxhSmYCr60upLK2waGEIifv3yv20NhsuXZCmtNRxIN1Cw7g7EG9eGv9fpo0RE8EUGkSEWFdQRl//3wXs8akkt2v51G3mZ2dTlVdI6+tLuzkdCIdo7ahiRe/zGfioF707hHudBzxcNOyEjlUVceXmgRHBFBpEhEfV9fYxM3zcoiLDOG2CwZ973bDU6IZkRrN88vz9MmrdEmLcoooqa5ndna601GkCzhrYC/Cgvx1o1sRF5UmEfFpT3+0g20Hqnjw4mFEHuMaj+tOTWdPSQ0fbTnYSelEOoa1ljlL8xgYF8GEvj2cjiNdQGiQP+dmxPHOhv00NDU7HUfEcSpNIuKzNhaV8+dPdjJjRBJnDTr2/WrOHxJPQlQIc5bu7oR0Ih1n5e5SNu2r4NrsNIzRNOPSPlMzEymraeCLHYecjiLiOJUmEfFJDU3N3Dwvl+iwIO6cmtGufQL9/bh6fBrLdpaweV+FmxOKdJw5S/OIDgvkouFJTkeRLuT0AT2JCAlgcc4+p6OIOE6lSUR80jOf7WJjUQX3XzSU6LCgdu83a0wKIYF+PKeb3UoXUVBaw3ub9jNrTCqhQf5Ox5EuJDjAn/OHxPPexv3UNjQ5HUfEUSpNIuJzth+o5A8fbGdKZgKThsYf177RYUHMGJHM6+v2UlJV56aEIh3nhRV7MMZw1bjeTkeRLmhaViKVdY18tq3Y6SgijlJpEhGf0tRsuWleLt1CArjnwiEndIzZE9Kob2xm7sr8Dk4n0rFq6ht5eWU+k4bGkxgd6nQc6YIm9O1BTFggi3I1RE98m0qTiPiUZ7/YzbqCMu6+cAg9uwWf0DH6x0VwWv+e/Gv5HuobNauUeK75a/dSUdvIddlpTkeRLirQ34/JwxL4YNMBauobnY4j4hiVJhHxGbsPVfP4e1s5NyOOaZkJJ3Ws605N52BlHe9s0Kev4pmamy3PLd1NZnIUI1JjnI4jXdjUzASONDTpdgvi09xamowxk4wxW40xO4wxtx5l/VPGmHWuxzZjTFmrddcYY7a7Hte0Wh5kjHnGtf0WY8wl7nwNIuIdmpstt8zLJTjAj/svGnrS0y6f0T+WPj3DefaL3Virm92K5/l8xyF2FlczW9OMy0kam96D2IhgzaInPs1tpckY4w88DUwGMoBZxphvzOtrrf21tXa4tXY48CdggWvf7sBdwFhgDHCXMebrj8nuAA5aawe4jvupu16DiHiPf3+5h5V5pfxuagZxkSEnfTw/P8Ps7DRyCstZm1928gFFOticpbuJjQhmyrBEp6NIF+fvZ5gyLIGPtx6ksrbB6TgijnDnmaYxwA5r7S5rbT3wMjC9je1nAXNdX58PvG+tLbXWHgbeBya51l0HPARgrW221uqOayLSpoLSGh5+ZwunD4hl5sjkDjvujBHJRIQE8KxudiseZmdxFZ9sLebKsb0JCtBIfDl507ISqGts5oPNB5yOIuIId/4lTQIKWn1f6Fr2HcaY3kA68FFb+xpjol3f32eMWWuMec0YE9ehqUXEq1hruW3Begzw0IxhHTpMKTw4gFljUnl3w36Kyo502HFFTtbzy/II8vfjirGpTkcRL3FKSgyJUSEaoic+y1M+frocmGetPdad0wKAZGCZtXYEsBx4/GgbGmOuN8asNsasLi7WvQVEfNWrqwv4YschbrtgMElumHL56vG9sdbyr+V7OvzYIiei/EgD89YUMi0rkdiIE5shUuTb/PwMUzIT+Gx7MeU1GqInvsedpWkvkNLq+2TXsqO5nP8bmtfWviVADa5rn4DXgBFHO6C19hlr7Shr7ajY2NjjTy8iXd7+8lruX7yZcX26c8UY93zinhwTxvlD4pm7Ml/T8YpHeG11ATX1TczWNOPSwaZlJdLQZFmycb/TUUQ6nTtL0yqgvzEm3RgTREsxWvjtjYwxg4AYWs4afW0JcJ4xJsY1AcR5wBLbMkXVIuBM13YTgU3uewki0lVZa7nj9fU0NlseuSQTPz/3zR42Ozud8iMNvP7V930uJNI5mpotzy3LY0xad4YmRTkdR7zMsKQoUruHsSi3yOkoIp3ObaXJWtsI/JyWArQZeNVau9EYc68x5sJWm14OvGxbzdlrrS0F7qOleK0C7nUtA7gFuNsYkwtcBdzortcgIl3XG+v28uGWg9x0/kB69wh363ONTothaFIkc5bmafpxcdQHmw9QePiIzjKJWxhjmJaVwLKdJRyqqnM6jkincus1Tdbat621A6y1fa21D7iW3WmtXdhqm7uttd+5h5O19llrbT/XY06r5XustadbazOttROttfnufA0i0vUcrKzl7oWbGNk7hmsmpLn9+YwxzJ6Qzo6DVXy+XRN6inPmLN1NUnQo52ZojiRxj6mZiTQ1W97ZoCF64ls8ZSIIEZEOc9ebGznS0MQjl2Ti78Zhea1NzUqgZ7dgTT8ujtm8r4IVu0q5enxvAvz1z7u4x6D4CPr16sbiHA3RE9+iv6oi4lXeXr+Pdzbs59fnDKBfr26d9rzBAf5cNa43n2wtZmdxVac9r8jX5izdTWigP5eP1jTj4j7GGKZmJrAyr5QDFbVOxxHpNCpNIuI1SqvrufPNDQxLiuLHp6V3+vNfMTaVIH8/nlua1+nPLb6tpKqON9YVMWNEElFhgU7HES83NTMRa+GtXN2zSXyHSpOIeI17F22k/EgDj12a6cjwpNiIYC4cnsi8NYW6j4l0qrkr86lvbNYEENIp+vXqxuCESM2iJz5FpUlEvMIHmw7wxroifnZWPwbFRzqWY3Z2Gkcamnhlteaokc7R0NTMCyv2cFr/nvTrFeF0HPER07IS+Cq/jMLDNU5HEekUKk0i0uWVH2ngjjfWMyg+gp+e2c/RLEMSoxib3p3nl+2hsanZ0SziG95ev48DFXVcl935Q1LFd00dlghoiJ74DpUmEenyHnxrM4eq6nlsZhZBAc7/WZudnc7esiO8v+mA01HEB8xZmkefnuGcMSDW6SjiQ1J7hJGVEq0heuIznH93ISJyEj7bVswrqwv4f6f3YVhylNNxADg3I47kmFBNPy5u91X+YdYVlHHNhDT8Oml6fZGvTctMYMPeCnYfqnY6iojbqTSJSJdVVdfIbQvW0zc2nP+Z2N/pOP/h72e4dkIaq/IOs76w3Ok44sXmLM0jIjiAS0YmOx1FfNAFwxIAdM8m8QkqTSLSZT3yzhaKyo/w6MwsQgL9nY7zDZeNTiE8yJ85OtskbrK/vJa31+/jstEpdAsOcDqO+KDE6FBGp8WwWNc1iQ9QaRKRLmnFrhJeWLGH67LTGdk7xuk43xEZEsjMkcksyi3iYKVuACkd798r9tBkLdeMT3M6iviwqZmJbD1QybYDlU5HEXErlSYR6XKO1Ddxy/xcevcI4zfnDXQ6zve6NjudhibLv1do+nHpWLUNTby0Mp9zBseR2iPM6TjiwyYPi8fPaIieeD+VJhHpcp54byt7Smp4eEYmoUGeNSyvtfSe4Zw9qBcvfbmH2oYmp+OIF1m4rojS6nrdzFYc1ysihHF9erAodx/WWqfjiLiNSpOIdClr8w/zz6W7uXJcKuP79nA6zjFdl53Ooap6FulTWOkg1lqeXbqbQfERjO/j+b8D4v2mZSWy+1A1G4sqnI4i4jYqTSLSZdQ2NHHzvFwSo0K5dfJgp+O0S3a/HgyI68acpXn6FFY6xIpdpWzZX8ns7DSM0TTj4rxJQ+IJ8DO6Z5N4NZUmEeky/vTRdnYcrOLBGcO6zGxhxhhmZ6ezaV8FX+4udTqOeIE5S3cTExbI9OFJTkcRASAmPIhT+/dkcY6G6In3UmkSkS5hfWE5f/10F5eNSuaMAbFOxzkuF5+SRExYoKYfl5NWUFrD+5sPcMXYVI+bZl9829TMRPaWHeGrgjKno4i4hUqTiHi8+sZmbpqXQ4/wIO6YkuF0nOMWEujPrDGpvLfpAAWlNU7HkS7s+WV5+BvDVePSnI4i8g3nDYkjyN+PxTm6Z5N4J5UmEfF4f/lkJ1v2V/LAxcOICg10Os4JuWp8b/yN4flleU5HkS6quq6RV1YXMHlYAvFRIU7HEfmGyJBAzhgYy1vri2hu1hA98T7tKk3GmF8aYyJNi38aY9YaY85zdzhvs+NgFfcs2qg/JiLHYcv+Cv734+1MH57IuRlxTsc5YQlRoUwelsArqwqoqmt0Oo50QfPXFlJZ26hpxsVjTctK5EBFHavydP2meJ/2nmm6zlpbAZwHxABXAQ+7LZWXWrOnlDlL83hxpW50KdIejU3N3Dwvl8iQQO6aNsTpOCftuuw0Kusambe6wOko0sU0N1ueW5pHVko0I1JjnI4jclQTB/UiJNCPxbkaoifep72l6es5TS8AXrDWbmy1TNrpslEpnNqvJw+/vZnCw7quQeRY/vHFbnILy7l3+lC6hwc5HeeknZIaw/CUaJ5blqczznJcPt1ezK5D1Vyns0ziwcKDA5g4OI631++jsanZ6TgiHaq9pWmNMeY9WkrTEmNMBKDfhuNkjOGhGcOwwO2vb9C0nCJt2FlcxZPvb2PSkHguGBbvdJwOc92p6eSV1PDx1oNOR5EuZM7SPOIig5k8NMHpKCJtmpaZQEl1PSt2aYieeJf2lqYfArcCo621NUAgMNttqbxYSvcwbp08iM+2FTNvTaHTcUQ8UlOz5eZ5uYQG+nPvRUO86gaek4fGEx8ZwpyleU5HkS5ix8EqPttWzFXjehMUoPmbxLOdObAX4UH+LMrRjW7Fu7T3r+94YKu1tswYcyXwW6DcfbG825VjezMmrTv3Ld7EgYpap+OIeJx/Lc9jzZ7D3DUtg14R3jVLWKC/H1eN780XOw6xdX+l03GkC3hu2W6CAvyYNSbV6SgixxQS6M95Q+J5d+N+6hs1KEm8R3tL01+AGmNMFnAjsBP4l9tSeTk/P8MjMzOpa2zmDg3TE/mGPSXVPPruVs4e1IuLT0lyOo5bXDEmleAAP55bppvdStvKaxqYv2YvFw1PpEe3YKfjiLTL1MwEyo808MWOYqejiHSY9pamRtvyzn468L/W2qeBCPfF8n7pPcO58bwBfLD5AIs0y4wI0DJD2K3z1xPgZ3jg4qFeNSyvtZjwIGaMSGLB2r2UVtc7HUc82Cur8znS0MTs7HSno4i022n9Y4kMCdCNbsWrtLc0VRpjbqNlqvG3jDF+tFzXJCfhh6f2ISslmrsXbqSkqs7pOCKOm7sqn+W7SrhjymASokKdjuNW105Ip66xmbm6BYF8j8amZp5ftodxfbozOCHS6Tgi7RYU4MekofG8t+kAtQ1NTscR6RDtLU3/BdTRcr+m/UAy8JjbUvkIfz/DYzMzqaxt4O5Fm5yOI+KovWVHeOjtLWT368F/jU5xOo7bDYyP4NR+PXlh+R4aNDWvHMUHmw+wt+yIzjJJlzQtK5GqukY+2aoheuId2lWaXEXpRSDKGDMVqLXW6pqmDjAgLoJfnN2fRTlFLNm43+k4Io6w1nL7gvU0W8vDMzK9dljet113ahr7K2p5Z4N+9+W7nl2aR0r3UM4ZHOd0FJHjNr5PD3qEB7EoV7PoiXdoV2kyxlwGrAQuBS4DvjTGzHRnMF/ykzP7Mjghkt++sYHymgan44h0uvlr9/LptmJumTSIlO5hTsfpNGcO6EV6z3Ce/UITQsg3bdhbzsrdpVwzPg1/P9/4EEG8S4C/H5OHxfPR5oPU1Dc6HUfkpLV3eN4dtNyj6Rpr7dXAGOB37ovlWwL9/XhsZial1fXc95aG6YlvOVhRy72LNjI6LYarxvV2Ok6n8vMzXDshjXUFZazNP+x0HPEgc5bmERbkz6WjvH+oqnivqZmJHGlo4oPNupm3dH3tLU1+1trWP/Elx7GvtMPQpCj++4w+zFtTyKfbNP5XfIO1ljve2EBdYzOPXJKJnw9+on7JyGQiggN0s1v5j+LKOhblFDFzZDJRoZpzSbqu0WndiYsMZrFudCteoL3F511jzBJjzLXGmGuBt4C33RfLN/3i7P7069WN2+bnUlmrYXri/Rbn7uP9TQe48bwB9Int5nQcR3QLDuC/Rqfw9vp97Cs/4nQc8QAvfZlPfVMz10xIczqKyEnx9zNcMCyBT7YVU6H3NdLFtXciiJuAZ4BM1+MZa+0t7gzmi0IC/Xnkkkz2VdTyyLtbnI4j4lYlVXXctXAjWSnR/PDUPk7HcdQ1E9Kw1vLC8j1ORxGHlVTV8dyy3Zw5MJa+PvpBgniXaVmJ1Dc28/7GA05HETkp7R5iZ62db629wfV43Z2hfNnI3jFcl53Ov1fks3xnidNxRNzmroUbqapt5LGZmT5/oXtK9zDOzYjjpZX5HKnXPU182d2LNlFV18htkwc7HUWkQ5ySEk1SdCiLNYuedHFtliZjTKUxpuIoj0pjTEVnhfQ1vzlvIL17hHHrgly9gRKv9O6G/SzO3cf/TOzHgLgIp+N4hNnZ6ZTVNPDGur1ORxGHLNm4n0U5Rfzi7P4MjNfvhXgHYwxTMxP4fPshDlfXOx1H5IS1WZqstRHW2sijPCKstce8PbkxZpIxZqsxZocx5tajrH/KGLPO9dhmjClrte4aY8x21+Oao+y70BizoZ2vs0sJDfLn4RmZ7Cmp4Yn3tjodR6RDldXU87s3N5CREMn/O6Ov03E8xtj07mQkRDJn6W6stU7HkU5WVlPPb9/YwOCESH5ypn4vxLtMy0qksdnqfpTSpbltBjxjjD/wNDAZyABmGWMyWm9jrf21tXa4tXY48CdggWvf7sBdwFhapje/yxgT0+rYM4Aqd2X3BOP79uDKcan8c+luTUUsXuW+xZs5XF3PY5dmEuivSTi/ZoxhdnYa2w5UsXSHhub6mnsXb6K0up7HZur3QrzPkMRI0nqE6Ua30qW58y/zGGCHtXaXtbYeeBmY3sb2s4C5rq/PB9631pZaaw8D7wOTAIwx3YAbgPvdltxD3DJpEAmRIdw8L5faBg3Tk67v460Hmb+2kJ+c2ZchiVFOx/E407IS6dktiGeX6ma3vuTjLQdZsHYvPzmjL0OT9Hsh3scYw7SsRJbvLKG4ss7pOCInxJ2lKQkoaPV9oWvZdxhjegPpwEft2Pc+4Amgpq0nN8Zcb4xZbYxZXVzcNe97FBESyIMzhrHjYBV/+mi703FETkplbQO3L1hP/17d+PnZ/ZyO45FCAv25YmxvPtpykN2Hqp2OI52goraB219v+b34xUT9Xoj3mpqZSLOFdzbsczqKyAnxlDEAlwPzrLVtnk4xxgwH+rZn9j5r7TPW2lHW2lGxsbEdFLPznTmwFzNHJvPXT3exYW+503FETthD72zhQEUtj87MJDjA3+k4HuvKcakE+hue09kmn/DQ25s5UFHLY5dm6fdCvNrA+AgGxHVjcY5Kk3RN7ixNe4GUVt8nu5YdzeX839C8tvYdD4wyxuQBXwADjDGfdFBej/W7KRl0Dw/ipnm5NDQ1Ox1H5Lgt23GIl77M50en9eGU1Jhj7+DDekWEMC0rkdfWFFJ+RDeD9GZfbD/E3JUF/Pi0PgxPiXY6jojbTc1MZGVeqW7kLV2SO0vTKqC/MSbdGBNESzFa+O2NjDGDgBhgeavFS4DzjDExrgkgzgOWWGv/Yq1NtNamAacC26y1Z7rxNXiEqLBAHrhoKJv3VfDXT3Y6HUfkuFTXNXLLglzSe4Zzw7kDnI7TJVyXnU5NfROvrS449sbSJVXXNXLrglz69Azn1/q9EB8xNTMBgLdydbZJuh63lSZrbSPwc1oK0GbgVWvtRmPMvcaYC1ttejnwsm01x661tpSWa5dWuR73upb5rPOGxDM1M4E/frSdbQcqnY4j0m6PLdlK4eEjPDozk5BADT9qj6FJUYxJ685zy/Joatb0497o0Xe3sLdMvxfiW/rEdmNIYiSLVJqkC3LrNU3W2rettQOstX2ttQ+4lt1prV3Yapu7rbXfuYeTtfZZa20/12POUdbnWWuHujO/p7nnwiFEhARy07xcvZGSLmFVXinPL8/jmvFpjE7r7nScLmV2dhqFh4/w/qYDTkeRDvblrhKeX76Ha8anMUq/F+JjpmUlklNQRkFpm/N5iXgcT5kIQtqhR7dg7r5wCDkFZTz7hS4SF89W29DELfNySYoO5abzBzodp8s5NyOOpOhQTT/uZY7UN3HL/FxSuody8yT9XojvmTKsZYjeYp1tki5GpamLmZaZwLkZcTz+3lZNSSwe7akPtrHrUDWPXJJJeHCA03G6nAB/P66Z0JuVu0s1c6YXeeK9reSV1PDIjEzCgvR7Ib4npXsYp6RGsyhHN7qVrkWlqYsxxnD/RUMJDvDjlnm5NGuYnnignIIy/v7ZLmaNSSG7X0+n43RZ/zUqlbAgf+YszXM6inSAtfmH+efS3VwxNpUJ+r0QHzY1M5FN+yrYWVzldBSRdlNp6oLiIkP47dQMVuaV8u8v9zgdR+Qb6hqbuGleDr0iQrjtgsFOx+nSosICuWREMotyiiiurHM6jpyE2oYmbnoth4TIEG6bPMjpOCKOmjIsAWPQPZukS1Fp6qIuHZnMaf178vA7W3QxpXiUpz/eybYDVTw4YyiRIYFOx+nyrs1Oo76pmRf1AUmX9scPt7OzuJqHLskkQr8X4uPio0IYndadRblFtJo8WcSjqTR1UcYYHpoxDAPc/vp6/dERj7CpqII/f7yDGackcfagOKfjeIW+sd04c2As/16RT11jk9Nx5ASsLyznb5/t4tKRyZwxINbpOCIeYVpmAjsOVrFVt1GRLkKlqQtLjgnj1gsG8/n2Q7y2utDpOOLjGpqauWleDtFhQdw5LcPpOF7luux0DlXVaShLF1Tf2PJ70SM8iN9O0e+FyNcmD0vAT0P0pAtRaerifjAmlbHp3bnvrU3sL691Oo74sGc+28XGogruv2go0WFBTsfxKqf170m/Xt14dulunVXuYv78yQ627K/kgYuHERWmYXkiX+vZLZgJfXtqiJ50GSpNXZyfn+GRSzJpaGrmt29omJ44Y/uBSv7wwXamZCYwaWi803G8jjGG2dlpbCyqYFXeYafjSDtt3lfB/360g+nDEzk3Q8NVRb5tWlYCe0pq2LC3wukoIsek0uQF0nqG85vzBvLB5oMs1H0PpJM1NVtumpdLeLA/91w4xOk4XmvGKclEhQYyRze77RIa/zNcNZC7p+n3QuRozh8ST4CfYVGu3ruI51Np8hKzs9MZnhLN3Qs3cqhKUxNL55mzdDfrCsq4+8Ih9OwW7HQcrxUa5M+sMaks2bhfM2Z2AX/7bBcb9lZw7/ShxIRruKrI0USHBXH6gFjeyt2n+06Kx1Np8hL+fobHZmZSXdfEXQs3Oh1HfMTuQ9U8tmQr5wyO48KsRKfjeL2rx/fGGMMLKzT9uCfbcbBluOoFw+K5YFiC03FEPNrUzAT2lh3hqwINPRbPptLkRfrHRfDLc/rzVu4+3t2g2WjEvZqbLbfMzyUowI8HLh6KMcbpSF4vMTqUSUPjmbsyn+q6RqfjyFF8PVw1LNifey4c6nQcEY93bkYcQQF+LNIseuLhVJq8zPWn9yEjIZLfvrGRspp6p+OIF3vxyz2s3F3K76ZmEBcZ4nQcn3FddjqVtY3MX6vbDHiiOUt381V+GXdPG0JshIarihxLREggZw2M5e31+2jSED3xYCpNXibQ34/HLs2krKae+xZvdjqOeKmC0hoeemcLpw+I5dKRyU7H8SkjUqPJSo7iuaV5ugbAw/zfcNVeTB+u4aoi7TUtK5GDlXWs3F3qdBSR76XS5IWGJEbxkzP7Mn9tIR9vPeh0HPEy1lpuf309BnhoxjANy+tkxhiuOzWdXYeq+XRbsdNxxKW52XLLvJbhqvdfpN8LkeNx9qBehAb6s1iz6IkHU2nyUj8/ux/9e3Xj9gXrqaxtcDqOeJFXVxfw+fZD3HbBYJKiQ52O45MmD02gV0Qwz2r6cY/x7y/3sDKvZbhqfJSGq4ocj7CgAM7JiOOdDftpbGp2Oo7IUak0eangAH8enZnJgYpaHnpni9NxxEvsL6/l/sWbGdenO1eMSXU6js8KCvDj6vG9+Xz7IbYfqHQ6js8rKK3hYQ1XFTkpUzMTKK2uZ9nOEqejiByVSpMXOyU1hh+ems5LX+azbOchp+NIF2et5Y7X19PQ3Mwjl2Ti56fhR06aNSaVoAA/5izLczqKT7PWcuuCXA1XFTlJZwyIJSI4gEU5GqInnkmlycvdcO5A0nqEcev89dTUa4piOXFvriviwy0Huen8QfTuEe50HJ/Xo1swFw9PYsHaQs2U6aCXVxWwdEeJhquKnKSQQH/OHRLHko37qWtscjqOyHeoNHm50CB/Hrkkk/zSGh5fss3pONJFFVfWcfeijYxIjebaCWlOxxGX2aemUdvQzNyVBU5H8UlFZUd44K3NjO/TQ8NVRTrAtMxEKmob+XybRseI51Fp8gFj+/Tg6vG9mbNsN2v2aDpPOX53LdxATX0Tj87Mwl/D8jzGoPhIJvTtwb+W59Ggi6c71dezSDY1Ww1XFekg2f16Eh0WqFn0xCOpNPmImycNIjEqlJvm5VLboNPe0n5vr9/H2+v386tz+tOvVzen48i3zM5OZ195LUs27nc6ik9ZsHYvn2wt5uZJA0ntEeZ0HBGvEBTgx6Qh8by/6YDeq4jHUWnyEd2CA3hoxjB2FVfzhw+3Ox1HuojD1fXc+eYGhiVFcf1pfZyOI0dx9qBe9O4RxrNfaPrxznKwopZ7Fm1kVO8Yrhmf5nQcEa8yLSuR6vomPt6i+0yKZ1Fp8iGnD4jlslHJPPPZLtYXljsdR7qAexZtpPxIA49dmkmAv/5ceCJ/P8O1E9JYm1/GuoIyp+N4PWstd7yxgbrGZh6dqWF5Ih1tbHp3enYLYpGG6ImH0bsgH3PHlAx6hAdx07wc6ht1DYR8vw82HeCNdUX87Kx+DIqPdDqOtGHmyGS6BQcwRze7dbtFuft4f9MBbjh3AH1iNVxVpKMF+PtxwbAEPtpykKo6zfornkOlycdEhQbywMXD2LK/kr98stPpOOKhyo80cMcb6xkUH8FPz+zndBw5hoiQQC4blcJbufvYX17rdByvVVJVx90LN5KVEs2PNFxVxG2mZiZS29DMh5sPOB1F5D9UmnzQuRlxTB+eyP9+vJ0t+yucjiMe6MG3NnOoqp7HZmYRFKA/E13BtRPSaLKWf6/Y43QUr3Xnwo1U1jbw2MxMzSIp4kajescQHxnCopx9TkcR+Q+9G/JRd00bQmRIIDfPy6VRUxVLK59vL+aV1QVcf3ofhiVHOR1H2im1RxjnDI7jxS/3aNYpN3h3wz7eyt3H/5zdnwFxEU7HEfFqfn6GKZkJfLatmPIjDU7HEQFUmnxW9/Ag7pk+hNzCcv6pWbfEpaqukVvnr6dPbDi/nNjf6ThynGZnp3G4poE31+11OopXKaup57dvbCQjIZL/PrOv03FEfMK0rETqm5p5T7dTEA+h0uTDpgxL4PwhcTzx/jZ2Flc5HUc8wKPvbqGo/AiPzcwkJNDf6ThynMb36cGg+Aie/SIPa63TcbzGvYs2UVZTz2OXZhKoWSRFOkVWchQp3UNZnKsheuIZ9NffhxljuG/6UEID/bllXi7NzXqT5cu+3FXCv5bvYfaEdEb27u50HDkBxhiuy05n64FKlu8scTqOV/hoywEWfLWXn57ZlyGJGq4q0lmMMUzNTOSLHYcora53Oo6ISpOv6xUZwp1TM1i95zD/Wp7ndBxxyJH6Jm6Zn0tq9zB+c/4Ap+PISbhweCLdw4N4VtOPn7SK2gZuX7CBAXHd+NnZmkVSpLNNzUygqdny7gYN0RPnqTQJM0YkccaAWB5dspWC0hqn44gDnnhvK3klNTxySSZhQQFOx5GTEBLozw/GpvLhloPkHap2Ok6X9uBbmzlYWctjM7MIDtBwVZHOlpEQSZ+e4SzK0Y1uxXkqTYIxhgdnDMPPGG5dkKtrIXzM2vzD/HPpbq4cl8r4vj2cjiMd4MpxvQnwMzy3LM/pKF3W59uLeXlVAT8+vQ9ZKdFOxxHxScYYpmYlsmJ3CQcrdA86cZZKkwCQFB3KbRcMYumOEl5ZVeB0HOkktQ1N3Dwvl8SoUG6dPNjpONJB4iJDmDIsgXlrCqms1XS9x6v1LJK/PkfDVUWcNC0zAWvh7fWaEEKc5dbSZIyZZIzZaozZYYy59SjrnzLGrHM9thljylqtu8YYs931uMa1LMwY85YxZosxZqMx5mF35vc1s0anMr5PDx54azP7yo84HUc6wZ8+2s6Og1U8OGMY3YI1LM+bXHdqOlV1jby6utDpKF3OI+9oFkkRT9E/LoJB8RGaRU8c57bSZIzxB54GJgMZwCxjTEbrbay1v7bWDrfWDgf+BCxw7dsduAsYC4wB7jLGxLh2e9xaOwg4Bcg2xkx212vwNX5+hocvGUZjs+WO1zdomJ6X27C3nL9+uotLRyZzxoBYp+NIB8tMjmZk7xieX5ZHk2bGbLcVu0p4YYVmkRTxJFMzE1i95zBFZfpAV5zjzjNNY4Ad1tpd1tp64GVgehvbzwLmur4+H3jfWltqrT0MvA9MstbWWGs/BnAdcy2Q7LZX4IN69wjnN+cP5KMtB3lznS689Fb1jc385rUceoQH8dspGcfeQbqk67LTyS+t4cPNB5yO0iVoFkkRzzQ1MxGAt3S2SRzkztKUBLS+OKbQtew7jDG9gXTgo/bua4yJBqYBH37PMa83xqw2xqwuLi4+kfw+69oJaYxIjebuRRsprqxzOo64wV8/3cmW/ZU8cPEwosICnY4jbnL+kDgSo0I0/Xg7Pf7eVvZoFkkRj5PWM5xhSVEsytWHueIcT5kI4nJgnrW2qT0bG2MCaDkr9Udr7a6jbWOtfcZaO8paOyo2VkOPjoe/n+HRmVnU1Ddx18INTseRDrZ1fyV/+mg7F2Ylcm5GnNNxxI0C/P24ekIaK3aVsqmowuk4Hm3NnsM8q1kkRTzWtKwEcgvL2VOiWymIM9xZmvYCKa2+T3YtO5rL+b+hee3Z9xlgu7X29ycfU46mX69u/Oqc/ry9fr9mrPEijU3N3Dwvh8iQQO6+cIjTcaQTXD46hdBAf+bobNP3aplFMkezSIp4sCmuIXqaEEKc4s7StArob4xJN8YE0VKMFn57I2PMICAGWN5q8RLgPGNMjGsCiPNcyzDG3A9EAb9yY3YBrj+tD0OTIrnzzQ0crq53Oo50gH98sZucwnLunT6U7uFBTseRThAdFsSMEUm8mVPEoSoNtz2aP3y4nZ3F1TykWSRFPFZSdCgje8foRrfiGLeVJmttI/BzWsrOZuBVa+1GY8y9xpgLW216OfCybTVVm7W2FLiPluK1CrjXWltqjEkG7qBlNr61rqnKf+Su1+DrAvz9ePSSLMpqGrhv8San48hJ2llcxZPvb2PSkHguGBbvdBzpRLOz06hvbOalL/OdjuJxcgvLeOazXVw2KpnTNYukiEebmpnAlv2V7DhY6XQU8UFuvabJWvu2tXaAtbavtfYB17I7rbULW21zt7X2O/dwstY+a63t53rMcS0rtNYaa+3gr6cqt9b+w52vwddlJEby07P6seCrvXy0RTNwdVVNzZab5+USGujPvRcNwRjjdCTpRP16RXD6gFheWLGH+sZmp+N4jPrGZm56LZee3YK4Q7NIini8KcMSMAYW5WiInnQ+T5kIQjzYz8/qx8C4CG5fsIGK2gan48gJ+NfyPNbsOcxd0zLoFRHidBxxwHXZaRRX1vHWeg1t+dr/fryDrQcqefDiYUSFahZJEU/XKzKEsendWZxbpHtJSqdTaZJjCgrw49GZmRysrOWhtzc7HUeOU35JDY++u5WzBsZy8SlHnfVffMDp/WPpExvOnKV5erMBbCqq4M8f7+DiU5KYOFizSIp0FVMzE9lZXM3mfRqiJ51LpUnaJSslmh+f1oe5KwtYuuOQ03GkHZqbLa+syueiPy8lwM/wwMXDNCzPh/n5GWZnp5NbWM6aPYedjuOohqZmbpqXQ3RYEHdO1bA8ka5k8tB4/P0Mi3XPJulkKk3Sbr8+dwDpPcO5dUEu1XWNTseRNuQWljHjL8u4Zf56+saG89pPxpMYHep0LHHYJSOSiAwJYM7SPKejOOqZz3axsaiC+y8aQoxmkRTpUnp0C2ZC3x4s0hA96WQqTdJuIYH+PDozk8LDR3hsyVan48hRHK6u5/bX1zP96aUUHj7Ck5dl8er/G8+g+Eino4kHCAsKYNaYVN7duJ+9ZUecjuOIbQcq+cMH25kyLIFJQxOcjiMiJ2BaViIFpUfILSx3Oor4EJUmOS6j07pzzfg0nl+ex6q8UqfjiEtTs+XFL/dw1hOf8MqqAmZPSOej35zBjBHJGpIn33DV+N5Ya/nX8jyno3S6pmbLTfNyCQ/2557purmzSFd1fkY8gf5G92ySTqXSJMftpvMHkhQdyi3zcqltaHI6js/7Kv8wF/95KXe8voGBcRG8/T+ncee0DCJDNBuYfFdyTBiThsYz98t8aup9a5jts1/sJqegjLsvHELPbsFOxxGRExQVFsgZA2J5a/0+mps1RE86h0qTHLfw4AAenpHJrkPV/P6D7U7H8VklVXXcMi+Xi/+8jAMVtfzh8uG8fP04BsZHOB1NPNx12elU1DYyf+1ep6N0ml3FVTz+3lbOzYjjwqxEp+OIyEmampnIvvJa1uT79sQ20nlUmuSEnNq/J5ePTuGZz3aSU1DmdByf0tTcMrTqrMc/Yf7aQq4/vQ8f3ngm04cnaSietMvI3jEMS4riuaW7feJT2uZmyy3zcwkO8OP+i4bq90TEC5yTEUdwgB+LNURPOolKk5yw26cMpldECDfPy6W+sdnpOD5hzZ5Spv3pC+58cyNDk6J455encfsFg+kWHOB0NOlCjDFcd2oaO4ur+Wx7sdNx3O5fy/NYlXeY303NIC5SN3cW8QbdggM4e1Av3lq/nyYf+PBHnKfSJCcsMiSQBy4eytYDlTz98Q6n43i14so6bnw1h0v+spzDNfU8fcUIXvzRWPrHaSienJgpwxKJjQj2+unHC0preOTdrZwxIJaZI5OdjiMiHWhaViKHqur4cleJ01HEB6g0yUmZODiOi09J4umPd7B5X4XTcbxOY1Mzc5bu5uwnPmFhzl5+cmZfPrjhDKZkJmiIkZyUoAA/rhrXm0+3FbPjYJXTcdzC2pZhef5+hodm6ObOIt7mrIG9CAvyZ1HuPqejiA9QaZKTdufUDKLDArl5Xi6NTRqm11G+3FXC1D99wT2LNjE8JZp3f3U6t0waRLiG4kkHuWJsKkH+fjy3bLfTUdxi7soClu0s4fYLBuvmziJeKDTIn3Mz4nhnwz4a9P5D3EylSU5aTHgQ904fyvq95fz9c+9889WZDlbU8quXv+K/nllBZW0jf71yBP+6bgx9Y7s5HU28TM9uwUwfnsj8NXspr2lwOk6H2lt2hAff3syEvj2YNSbF6Tgi4iZTMxMpq2lg6Y5DTkcRL6fSJB3igmEJTB4az1MfbPPaoT7u1tDUzD8+38XZT3zK2+v384uz+/HBDWcwaaiG4on7zM5O50hDEy+vync6Soex1nL7gvU0NVseuSRTvz8iXuz0AT2JCAlgUY6G6Il7qTRJh7ln+hDCgvy5eV6OZrI5Tst3ljDlj59z/1ubGZ0Ww3u/Pp0bzxtIaJC/09HEy2UkRjKuT3eeX5bnNcNr560p5NNtxdwyaSAp3cOcjiMibhQc4M/5Q+J5b+N+6hqbnI4jXkylSTpMr4gQ7pqWwdr8Mp5flud0nC5hf3ktv5j7FbP+voKa+ib+fvUonr12NGk9w52OJj5kdnY6ReW1vLfpgNNRTtqBilruW7yJ0WkxXD0+zek4ItIJpmYmUFnXyKdbvf8WCuIclSbpUBcNT+KsgbE8tmQr+SU1TsfxWPWNzfz1052c/cQnLNm4n19O7M8HN5zBuRlxGkokne6cwXGkdA/l2S+69jWJ1lrueH0DdY3NPDozCz8//S6J+ILsfj2JCQtksWbREzdSaZIOZYzhwRnDCPAz3DI/F2s1TO/bvth+iMl/+IyH39nChL49+eDXZ/DrcwcQEqiheOIMfz/DNePTWL3nMLmFZU7HOWELc4r4YPMBfnPeQNJ1tlbEZwT6+zFpaAIfbD7AkXoN0RP3UGmSDpcQFcrtUwazfFcJc1cWOB3HYxSVHeGnL67hyn9+SWOzZc61o/nHNaNI7aFrLsR5l41OITzIv8ve7PZQVR13L9zI8JRorjs13ek4ItLJpmUlUFPfxEdbDjodRbyUSpO4xeWjU8ju14MH395MUdkRp+M4qq6xiac/3sHEJz7loy0HufHcASz51emcNaiX09FE/iMyJJBLR6WwOLeIgxW1Tsc5bne9uZHquiYem5mJv4blificsek9iI0IZlFOkdNRxEupNIlbGGN4eEYmTc2W219f77PD9D7ZepBJv/+cx5Zs5fQBPfnghjP4xcT+GoonHumaCWk0Nlv+vWKP01GOyzvr9/HW+n388pz+9I+LcDqOiDjA388wZVgCH289SFVdo9NxxAupNInbpHQP4+ZJA/lkazGvf7XX6TidqqC0huv/tZpr56zCAM9fN4a/XTWK5BgNxRPPld4znImDevHil/nUNnSN6wIOV9fzuzc3MCQxkutP7+N0HBFx0NTMBOoam/nAC2YCFc+j0iRudc34NEb1juGeRZs4WNn1hvwcr9qGJv744XbOefJTPt9+iJsnDeSdX53GGQNinY4m0i6zs9Mpqa5nYRcZ4nLv4k2U1TTw2MwsAv31T5qILxuRGkNiVIiG6Ilb6F8YcSs/P8MjMzM50tDE797Y4NXD9D7acoDzf/8ZT76/jXMGx/HhjWfw0zP7ERygoXjSdUzo24OBcRE8+8Vuj/99/XDzAV7/ai8/PasfGYmRTscREYf5+RmmZCbw2fZiymsanI4jXkalSdyub2w3bjh3AEs2HuDt9fudjtPh8ktq+OFzq7juudUE+Bn+/cOxPP2DESRGhzodTeS4GWOYnZ3Glv2VrNhV6nSc71V+pIHbX1/PwLgIfn5WP6fjiIiHmJaVSEOTZclG73u/Ic5SaZJO8aNT08lMjuLONzdQWl3vdJwOUdvQ1HJW6alPWbGrhNsvGMQ7vzydU/v3dDqayEm56JQkYsICeXap597s9oG3NnGoqp7HLs0kKED/lIlIi2FJUaR2D2NRroboScfSvzTSKQL8/Xh0ZiYVtQ3cu2ij03FOirWW9zbu55wnP+WPH25n0pB4PrzxTK4/va/evIlXCAn054qxqXyw+QD5JTVOx/mOz7YV8+rqQq4/vQ+ZydFOxxERD2KMYVpWAst2llBSVed0HPEieocnnWZQfCQ/O6sfb6wr6rIz2+w+VM3s51Zx/QtrCAvyZ+6Px/HHWacQHxXidDSRDnXVuDT8jeG5ZXlOR/mGqrpGbluwnr6x4fxyYn+n44iIB5qamUhTs+WdDRqiJx1HpUk61U/P7Meg+AjueGM95Ue6zkWaNfWNPLZkC+c/9Rmr8w7z2ymDeet/TmN83x5ORxNxi/ioEC4YlsCrqwuorPWc39WH39lMUfkRHp2ZpfudichRDYqPoG9suGbRkw6l0iSdKijAj8dmZnGoqp4H39rsdJxjstbyzvp9nPPEpzz98U6mZibw0W/O4Een9dH0xuL1rjs1naq6RuatKXQ6CgDLd5bw7xX5XJedzsjeMU7HEREP1TJEL5GVeaUcqPD+251I59C7Pul0w5KjuP70PryyuoDPtxc7Hed77ThYxdXPruQnL64lMjSQ1/57PE/+13B6RWgonviG4SnRnJIazXPL8mhudnb68Zr6Rm6Zn0vvHmH85ryBjmYREc83NTMRa+Gt3H1ORxEvodIkjvjlxP70iQ3n1vnrqa5rdDrON1TXNfLQO5uZ/IfPWFdQxt3TMlj8i1MZndbd6Wgine667HT2lNTw0ZaDjuZ4fMk28ktreOSSTEKDNCxPRNrWr1c3BidEsliz6EkHUWkSR4QE+vPoJZkt1ya8u8XpOEDLULxFOUVMfOJT/vbpLqYPT+KjG8/k2ux0AjQUT3zUpKHxxEeGODr9+Oq8UuYs283V43szro+uIxSR9pmamcDa/DIKD3veLKDS9eidoDhmVFp3rp2QxvPL97Byt7M30dx+oJIf/ONLfjH3K3p0C2L+Tybw+KVZxEYEO5pLxGmB/n5cPaE3y3aWsGV/Rac/f21DEzfPyyUxKpSbJw3q9OcXka5rWmYioCF60jFUmsRRN50/kJTuodwyP5fahqZOf/7K2gbuX7yJyX/4nI1FFdx30VAW/vxUXWQu0sqs0amEBPox54u8Tn/upz7Yxq5D1Tx8yTC6BQd0+vOLSNeV2iOMrOQo3ehWOoRbS5MxZpIxZqsxZocx5tajrH/KGLPO9dhmjClrte4aY8x21+OaVstHGmPWu475R2OMcedrEPcKCwrgkRmZ7D5UzVPvb+u057XW8sZXe5n4xKf8c+luZo5M5uPfnMlV43rj76cfKZHWYsKDuPiUZF5ft7dTbxaZU1DG3z/bxeWjUzitf2ynPa+IeI9pWYls2FtB3qFqp6NIF+e20mSM8QeeBiYDGcAsY0xG622stb+21g631g4H/gQscO3bHbgLGAuMAe4yxnz90f9fgB8D/V2PSe56DdI5JvTryawxqfz9812sKyhz+/Nt2V/Bfz2zgl+9so74qBBe/2k2D1+SSffwILc/t0hXdV12GvWNzcxdmd8pz1fX2MRN83LoFRHC7VMGd8pzioj3uWBYAoAmhJCT5s4zTWOAHdbaXdbaeuBlYHob288C5rq+Ph9431pbaq09DLwPTDLGJACR1toV1loL/Au4yG2vQDrNbRcMIi4yhJvn5VDX6J5hehW1DdyzaCNT/vgF2w9U8tCMYbz+02yGp0S75flEvEn/uAhO69+TF1bsob6x2e3P9/RHO9h2oIoHZwwlMiTQ7c8nIt4pMTqU0WkxLMrRdU1yctxZmpKAglbfF7qWfYcxpjeQDnx0jH2TXF+355jXG2NWG2NWFxd77r2ApEVkSCAPXjyMbQeqePqjHR167OZmy7w1hZz9+Cc8tyyPy0en8NGNZzJrTKqG4okch+uy0zlQUcc7G9z75mNjUTl//mQnM05J4uxBcW59LhHxflMzE9l6oJJtByqdjiJdmKdMBHE5MM9a22GnGKy1z1hrR1lrR8XGaix8V3DWoF7MGJHEnz/Zyaaijpmla2NROZf+bTm/eS2H5JgwFv7sVB64eBgxGoonctzOGBBLn57hPPvFblpO9ne8hqZmbnotl+iwIO6clnHsHUREjmHysHj8DCzO0RA9OXHuLE17gZRW3ye7lh3N5fzf0Ly29t3r+ro9x5Qu6M6pGUSHBXHTvBwamk58CFB5TQN3vrmBaX/6gt2Hqnl0ZiYLfjKBYclRHZhWxLf4+RmuzU4jp7CctfllbnmOv326k037Krj/oqFEh+nDDRE5eb0iQhjXpweLc/e57QMf8X7uLE2rgP7GmHRjTBAtxWjhtzcyxgwCYoDlrRYvAc4zxsS4JoA4D1hird0HVBhjxrlmzbsaeNONr0E6WXRYEPdfNISNRRU889mu496/udny6qoCzn7iE/69Yg9XjevNxzeeyWWjUvDTUDyRk3bJiGQiQgKY44ab3W47UMkfP9zB1MwEJg2N7/Dji4jvmpaVyK5D1WzsoJEs4nvcVpqstY3Az2kpQJuBV621G40x9xpjLmy16eXAy7ZV9bfWlgL30VK8VgH3upYB/BT4B7AD2Am8467XIM6YNDSBKcMS+MMH29lxsP3jj9cXljPjL8u4eX4u6T3DWfSLU7ln+lCiwnQRuUhHCQ8O4PLRKbyzYT9FZUc67LiNTc3c9FoO3UICuOfCIR12XBERgElD4gnwMyzWjW7lBBlfOE05atQou3r1aqdjyHEorqzjvKc+Ja1nOPP+e0KbEzYcrq7nsfe2MndlPj3Cg7lt8iBmjEhCt/AScY+C0hrOeOxj/t8Zfbll0qAOOebfPt3JQ+9s4U+zTmFaVmKHHFNEpLVrnl3JzuIqPr/5LL1HkKMyxqyx1o462jpPmQhC5BtiI4K5+8IhfJVf9r3DgJqaLS99mc9ZT3zCK6sKmD0hnY9+cwaXjEzWH0MRN0rpHsZ5GfG89GU+R+pPfv6encVVPPH+Ns7LiGNqZkIHJBQR+a5pWYkUHj7SKfeEFO+j0iQe68KsRM4Z3IvH39vKnpJv3sl7XUEZF/95Kbe/vp4BcRG89T+ncue0DN3PRaSTzM5Oo/xIAwu+Kjz2xm1oarbcMi+X0EB/7r9oqD7wEBG3OW9IHEH+frpnk5wQlSbxWMYY7r9oGIF+ftwyP5fmZktJVR23zs/l4j8vZX95LX+4fDivXD+OQfGRTscV8Slj0rszJDGSOUvzTmo2qn8tz2P1nsPcOTWDXpEhHZhQROSbIkMCOWNgLG+tL6K52fsvT5GOpdIkHi0+KoTfTh3Mil2l/OqVdZz9xKfMW1PIj05N58Mbz2D6cF27JOIEYwyzs9PZcbCKz7cfOqFj7Cmp5tF3t3LmwFhmjDjqfcpFRDrU1MwEDlTUsSqv9Ngbi7Si0iQe77JRKZzarycLc4rISIjknV+exh1TMojQUDwRR03LSqBnt6ATmn68udly6/z1BPgZHpoxTB9+iEinOGdwHCGBfppFT46bSpN4PGMMT/9gBHN/PI6XfjyW/nERTkcSESA4wJ8rx/Xm463F7CyuOq59X1qZz/JdJdw+ZTAJUaFuSigi8k3hwQFMHBTHOxv20djU7HQc6UJUmqRLiAoNZHzfHvo0WsTD/GBsb4L8/Xh+WV6799lbdoSH3t5Mdr8eXD46xX3hRESOYlpWAoeq6lmxS0P0pP1UmkRE5ITFRgQzLSuReWsKKT/ScMztrbXctmA9Fnh4RqY+CBGRTnfmwF6EB/mzOLfI6SjShag0iYjISZmdnUZNfROvrio45ravrSnks23F3Dp5ECndwzohnYjIN4UE+nPekHje2bCf+kYN0ZP2UWkSEZGTMjQpijHp3XluWV6b1wgcqKjlvsWbGJPenSvH9u7EhCIi3zQ1M4HyIw0s3XFis3+K71FpEhGRk3Zddhp7y47wweYDR11vreWO19fT0NTMo5dk4uenYXki4pzT+scSGRLAohwN0ZP2UWkSEZGTdm5GPMkxoTz7Rd5R1y/MKeKDzQf5zXkDSesZ3rnhRES+JSjAj0lD43lv0wFqG5qcjiNdgEqTiIicNH8/wzXj01iZV8qGveXfWFdcWcddCzdySmo0s7PTHUooIvJNUzMTqapr5JOtxU5HkS5ApUlERDrEZaNTCAvy59lv3ez2roUbqKlv4rGZmfhrWJ6IeIgJfXvQPTxIs+hJu6g0iYhIh4gKDWTmyGQW5RRxsLIWgLfX7+Pt9fv51Tn96ddLN6YWEc8R4O/H5KHxfLj5IDX1jU7HEQ+n0iQiIh3m2glpNDRZXlyRT2l1PXe+uYFhSVFcf1ofp6OJiHzHtKxEjjQ08eHmg05HEQ8X4HQAERHxHn1iu3HWwFhe/HIPOw5WUX6kgRd+OJYAf31GJyKeZ3Rad3pFBLMop4hpWYlOxxEPpn/FRESkQ113ajqHqup5a/0+fnZWPwYnRDodSUTkqPz9DFMyE/hkWzGVtQ1OxxEPptIkIiId6tR+PRmaFElGQiQ/PbOf03FERNo0NTOR+sZm3t909PvMiYCG54mISAczxvDK9eMJ8DcEBeizORHxbCNSo0mKDmVRThEzRiQ7HUc8lP41ExGRDhceHEBwgL/TMUREjskYw9TMBD7ffoiymnqn44iHUmkSEREREZ82LSuRxmbLuxv2Ox1FPJRKk4iIiIj4tCGJkaT1CGNx7j6no4iHUmkSEREREZ9mjGFaViLLdh6iuLLO6TjigVSaRERERMTnTc1MpNnCuxt0tkm+S6VJRERERHzewPgIBsR1Y1GOSpN8l0qTiIiIiAgtZ5tW7SllX/kRp6OIh1FpEhEREREBpmYmYC28pQkh5FtUmkREREREgD6x3RiSGKlZ9OQ7VJpERERERFymZiayrqCMgtIap6OIB1FpEhERERFxmZqZAKCzTW7U2NTMB5sO8KPnV3OwotbpOO0S4HQAERERERFPkdI9jOEp0SzKKeInZ/Z1Oo5X2Vt2hFdWFfDqqgL2V9QSGxHMzuJqekWGOB3tmFSaRERERERamZaVyH2LN7GruIo+sd2cjtOlNTY189GWg8xdmc8n24oBOL1/LHdfOISJg3sR6N81Br6pNImIiIiItDJlWAL3v7WJxbn7+J+J/Z2O0yUVlNa0nFVaXcDByjriIoP5xVn9uHRUCindw5yOd9xUmkREREREWomPCmF0WncW5RSpNB2HhqZmPtx8gJdWFvD59mIMcObAXswak8pZA2MJ6CJnlY5GpUlERERE5FumZSbwuzc3snV/JQPjI5yO49HyS2p4eVU+r64u5FBVHQlRIfzP2f25bHQKSdGhTsfrEG4tTcaYScAfAH/gH9bah4+yzWXA3YAFcqy1V7iWPwJMcW12n7X2FdfyicBjtMz8VwVca63d4c7XISIiIiK+ZfKwBO5auJFFOUUMjB/odByPU9/YzPubDjB3ZT5f7DiEn4GzB8Uxa0wKZwzo2meVjsZtpckY4w88DZwLFAKrjDELrbWbWm3TH7gNyLbWHjbG9HItnwKMAIYDwcAnxph3rLUVwF+A6dbazcaYnwK/Ba511+sQEREREd/Ts1swE/r2ZHFuETeeNwBjjNORPMLuQ9W8vCqfeasLKamuJyk6lBvOHcClo5JJiPKOs0pH484zTWOAHdbaXQDGmJeB6cCmVtv8GHjaWnsYwFp70LU8A/jMWtsINBpjcoFJwKu0nJGKdG0XBRS58TWIiIiIiI+alpXALfPXs2FvBcOSo5yO45i6xibe29hyVmnZzhL8/QwTB/Vi1thUTu8fi7+f9xdKd5amJKCg1feFwNhvbTMAwBizlJYhfHdba98FcoC7jDFPAGHAWfxf2foR8LYx5ghQAYxz2ysQEREREZ91/pB47nh9A4tzi3yyNO0sruLllfnMX7uX0up6kmNCuen8gVw6MrlL3FupIzk9EUQA0B84E0gGPjPGDLPWvmeMGQ0sA4qB5UCTa59fAxdYa780xtwEPElLkfoGY8z1wPUAqamp7n4dIiIiIuJlosOCOK1/Txbn7uPWyYN8YohebUMTSzbu56Uv8/lydykBfoZzM+KYNSaVU/v1xM8HziodjTtL014gpdX3ya5lrRUCX1prG4DdxphttJSoVdbaB4AHAIwxLwHbjDGxQJa19kvX/q8A7x7tya21zwDPAIwaNcp2zEsSEREREV8yLSuRG17NYW1+GSN7xzgdx222H6hk7soCFnxVSFlNA6ndw7hl0iBmjkwmNiLY6XiOc2dpWgX0N8ak01KWLgeu+NY2bwCzgDnGmJ60DNfb5ZpEItpaW2KMyQQygfdc+0QZYwZYa7fRMsnEZje+BhERERHxYedmxBEU4MeinCKvK021DU28vX4fc1fmsyrvMIH+hvOGxHPFmFTG9+nhs2eVjsZtpcla22iM+TmwhJbrlZ611m40xtwLrLbWLnStO88Ys4mW4Xc3uYpSCPC56xRoBXCla1IIjDE/BuYbY5qBw8B17noNIiIiIuLbIkICOWtgLG+v38fvpmZ4xaQHW/dXMndlPgvWFlJR20hajzBumzyIS0Ym07ObziodjbHW+0eujRo1yq5evdrpGCIiIiLSBS3KKeIXc79i7o/HMb5vD6fjnJAj9U0szi3i5VUFrNlzmCB/PyYNjWfWmFTG9enuE9drHYsxZo21dtTR1jk9EYSIiIiIiEebOLgXoYH+LM4t6nKlafO+CuauzOf1r/ZSWdtIn9hwfjtlMDNGJNM9PMjpeF2GSpOIiIiISBvCggKYOLgX72zYzz0XDiHA38/pSG2qqW9kcc4+XlqZz7qCMoIC/LjAdVZpTLrOKp0IlSYRERERkWOYlpXI4tx9LNtZwukDYp2Oc1Qb9pbz8qp83viqiKq6Rvr36sadUzOYMSKJ6DCdVToZKk0iIiIiIsdwxoBYIoIDWJxb5FGlqaqukUU5RcxdmU9uYTnBAX5MyUzgijGpjOwdo7NKHUSlSURERETkGEIC/Tl3SBzvbtjP/RcNIyjA2SF66wvLeWllPgvX7aW6vomBcRHcPS2Di09JJios0NFs3kilSURERESkHaZlJrJg7V4+317MxMFxnf78lbUNvLmu5azSxqIKQgL9mJaZyKyxqZySEq2zSm6k0iQiIiIi0g7Z/XoSHRbIopyiTitN1lpyCsuZ+2U+C3OKONLQxKD4CO6bPoTppyQRGaKzSp1BpUlEREREpB2CAvyYNCSeRTlF1DY0ERLo77bnqqht4M2v9vLSygI276sgLMif6cMTuXxMKlnJUTqr1MlUmkRERERE2mlaViIvryrg4y0HmTwsoUOPba1lbX4Zc1fmszi3iNqGZoYkRvLAxUO5MCuRCJ1VcoxKk4iIiIhIO41N707PbkEszt3XYaWpvKaB178qZO7KArYeqCQ8yJ+LT0nmijGpDEuO6pDnkJOj0iQiIiIi0k4B/n5MHprAa2sKqK5rJDz4xN5OW2tZs+cwL63M563cfdQ1NpOVHMXDM4YxLSvxhI8r7qH/GyIiIiIix2FaViIvrNjDB5sPMH140nHtW1ZTz/y1e5m7Mp8dB6voFhzApaOSuXx0KkOTdFbJU6k0iYiIiIgch1G9Y4iPDGFRzr52lSZrLSt3lzJ3ZT5vb9hPfWMzp6RG8+jMTKZmJhAWpLfknk7/h0REREREjoOfn2FKZgIvLN9D+ZEGokKPPkFDaXU9C9YW8tLKfHYVVxMREsCs0SlcPiaVwQmRnZxaToZKk4iIiIjIcZqamcA/v9jN+5sOMHNk8n+WW2tZvquEuSsLWLJhP/VNzYzsHcPjl/ZjyrAEQoPcN025uI9Kk4iIiIjIcRqeEk1yTCiLcoqYOTKZQ1V1zF9TyMurCth9qJrIkAB+MC6VWWNSGRAX4XRcOUkqTSIiIiIix8kYw9TMRP7x+S5++uIa3t90gIYmy5i07vzPxH5MHprg1pvfSudSaRIREREROQHThyfyt892smxnCVePT2PWmBT69dJZJW+k0iQiIiIicgIGJ0TywQ1nkBQdqrNKXk6lSURERETkBPWN7eZ0BOkEfk4HEBERERER8WQqTSIiIiIiIm1QaRIREREREWmDSpOIiIiIiEgbVJpERERERETaoNIkIiIiIiLSBpUmERERERGRNqg0iYiIiIiItEGlSUREREREpA0qTSIiIiIiIm1QaRIREREREWmDSpOIiIiIiEgbVJpERERERETaoNIkIiIiIiLSBpUmERERERGRNhhrrdMZ3M4YUwzscTqHS0/gkNMhxOfo506coJ87cYJ+7sQJ+rnzDr2ttbFHW+ETpcmTGGNWW2tHOZ1DfIt+7sQJ+rkTJ+jnTpygnzvvp+F5IiIiIiIibVBpEhERERERaYNKU+d7xukA4pP0cydO0M+dOEE/d+IE/dx5OV3TJCIiIiIi0gadaRIREREREWmDSpOIiIiIiEgbVJo6kTFmkjFmqzFmhzHmVqfziPczxqQYYz42xmwyxmw0xvzS6UziG4wx/saYr4wxi53OIr7BGBNtjJlnjNlijNlsjBnvdCbxfsaYX7v+fd1gjJlrjAlxOpO4h0pTJzHG+ANPA5OBDGCWMSbD2VTiAxqBG621GcA44Gf6uZNO8ktgs9MhxKf8AXjXWjsIyEI/f+Jmxpgk4H+AUdbaoYA/cLmzqcRdVJo6zxhgh7V2l7W2HngZmO5wJvFy1tp91tq1rq8raXkTkeRsKvF2xphkYArwD6eziG8wxkQBpwP/BLDW1ltryxwNJb4iAAg1xgQAYUCRw3nETVSaOk8SUNDq+0L05lU6kTEmDTgF+NLhKOL9fg/cDDQ7nEN8RzpQDMxxDQv9hzEm3OlQ4t2stXuBx4F8YB9Qbq19z9lU4i4qTSI+wBjTDZgP/MpaW+F0HvFexpipwEFr7Rqns4hPCQBGAH+x1p4CVAO6dljcyhgTQ8uooXQgEQg3xlzpbCpxF5WmzrMXSGn1fbJrmYhbGWMCaSlML1prFzidR7xeNnChMSaPlmHIZxtj/u1sJPEBhUChtfbrM+nzaClRIu50DrDbWltsrW0AFgATHM4kbqLS1HlWAf2NMenGmCBaLhRc6HAm8XLGGEPLGP/N1tonnc4j3s9ae5u1Ntlam0bL37mPrLX65FXcylq7Hygwxgx0LZoIbHIwkviGfGCcMSbM9e/tRDQBidcKcDqAr7DWNhpjfg4soWV2lWettRsdjiXeLxu4ClhvjFnnWna7tfZt5yKJiLjFL4AXXR9M7gJmO5xHvJy19ktjzDxgLS2z1X4FPONsKnEXY611OoOIiIiIiIjH0vA8ERERERGRNqg0iYiIiIiItEGlSUREREREpA0qTSIiIiIiIm1QaRIREREREWmDSpOIiMhRGGPONMYsdjqHiIg4T6VJRERERESkDSpNIiLSpRljrjTGrDTGrDPG/M0Y42+MqTLGPGWM2WiM+dAYE+vadrgxZoUxJtcY87oxJsa1vJ8x5gNjTI4xZq0xpq/r8N2MMfOMMVuMMS8aY4xjL1RERByj0iQiIl2WMWYw8F9AtrV2ONAE/AAIB1Zba4cAnwJ3uXb5F3CLtTYTWN9q+YvA09baLGACsM+1/BTgV0AG0AfIdvNLEhERDxTgdAAREZGTMBEYCaxynQQKBQ4CzcArrm3+DSwwxkQB0dbaT13LnwdeM8ZEAEnW2tcBrLW1AK7jrbTWFrq+XwekAV+4/VWJiIhHUWkSEZGuzADPW2tv+8ZCY373re3sCR6/rtXXTejfTRERn6TheSIi0pV9CMw0xvQCMMZ0N8b0puXft5muba4AvrDWlgOHjTGnuZZfBXxqra0ECo0xF7mOEWyMCevMFyEiIp5Nn5iJiEiXZa3dZIz5LfCeMcYPaAB+BlQDY1zrDtJy3RPANcBfXaVoFzDbtfwq4G/GmHtdx7i0E1+GiIh4OGPtiY5YEBER8UzGmCprbTenc4iIiHfQ8DwREREREZE26EyTiIiIiIhIG3SmSUREREREpA0qTSIiIiIiIm1QaRIREREREWmDSpOIiIiIiEgbVJpERERERETa8P8BnAKwNfIpXAEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1008x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(14, 7))\n",
    "plt.title(\"Loss VS Epoch\")\n",
    "\n",
    "plt.plot(train_loss_by_epoch, label=\"train_loss\")\n",
    "# plt.plot(dev_loss_by_epoch, label=\"dev_loss\")\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0YAAAG5CAYAAACqfyT9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlUUlEQVR4nO3dfdReZX0n+u/PBEypVlRClxIq6Rh86VQQHugLra+j0g6F1hGIp1VxespplR7bU+3gtKcqo+u0p472zJqMDrb2qKNGpdWJnFa0+FrrC08sqATRiFoetCXlxdciIL/zx31Hb2MiAXLnecL1+ax1r9z72tfe+7ez9tpPvrmuvZ/q7gAAAIzsHstdAAAAwHITjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAD7SVUdVVVdVauXuxYAvptgBMAdVlWfr6p/s5+PeW5VvX837YdV1c1V9a+r6uCq+s9VtVRVX5vW+SffZ59dVV+f9t35+d25nggAK5L/sQLgQPE/kry4qtZ39+dm2jcm+UR3f7KqXpBkIcmJSb6U5EFJHnU7+z2mu7fPpWIADhhGjADYZ6rqnlX1J1X1xennT6rqntN1h1XVhVV1Y1VdX1UfqKp7TNf9h6q6pqq+WlVXVtXjd913dy8leXeSp+2y6ulJXjv9fkKSt3b3F3vi89392twJVfXCqrqgqt40retjVXXMzPqHVdV7p+dzeVWdOrPuB6YjV1+oqi9X1d9W1Q/M7P6Xq+ofquqfq+r37kx9AOxbghEA+9LvJfnJJMcmOSaTkZvfn677nSRLSdYm+eEk/zFJV9VDkpyT5ITuvneSJyX5/B72/5rMBKPptscmecO06cNJ/o+qelZV/XhV1V08n9OSvCXJ/abHeFtVHVRVByV5e5J3Jjk8yW8mef20niR5aZLjk/z0dNvfTXLbzH5/JslDkjw+yR9U1cPuYp0A3EWCEQD70i8nOa+7r+3uHUlelO8EmVuSPCDJg7r7lu7+QHd3km8luWeSh1fVQdNRns/uYf9vTfLDVfXT0+WnJ/nr6bGS5P9K8kfTOhaTXFNVz7idmj82HfXZ+XnSzLqt3X1Bd9+S5GVJ1mQS/H4yyb2S/GF339zd705yYZKnTkfB/n2S53T3Nd39re7+u+7+5sx+X9Td/9LdlyW5LJMQCcAyEowA2JcemOQLM8tfmLYlyR8n2Z7knVV1VVWdmyTT53t+K8kLk1xbVZur6oHZje7+RiYjOE+fjgb9cr4zjS7TELKpu09KcmiSlyR59e2MyBzX3YfOfC6aWXf1zL5vy2TE64HTz9XTttlzPSLJYZkEqD2FuyT5x5nv38gkZAGwjAQjAPalL2bywoOdfmTalu7+anf/Tnf/aJJTM5ny9vjpujd0989Mt+1MRn325DVJzkjyhCT3zmRK2/eYjshsSnJDkoffyfM5cueX6UjQuun5fDHJkTufkZr6kSTXJPnnJDcl+Vd38pgALAPBCIA766CqWjPzWZ3kjUl+v6rWVtVhSf4gk7fJpapOqaoHT0d6vpzJFLrbquohVfW46UsabkryL/nu53F29YEkNyY5P8nm7r5554qq+q2qesz05Qerp9Po7p3k7+/kOR5fVU+enttvJflmJs8xfSSTkZ7fnT5z9JgkvzCt57Ykr07ysqp6YFWtqqqf2vkSCgBWJsEIgDvrrzIJMTs/L0zy4kye7fl4kk8k+di0LUk2JPmbJF9L8qEk/62735PJ80V/mMlIyz9m8jKD5+/poNPnkl6byejSrm+c+0aS/zzdzz8neXaSf9fdV32f87hsl99j9Ccz6/5nkjMzGXV6WpInT5+PujmTIPRz0+P8tyRP7+5PTbd77vT8L0lyfSYjYH7mAqxgNfn5AgDMqqoXJnlwd//KctcCwPz53ysAAGB4ghEAADA8U+kAAIDhGTECAACGt3q5C9hXDjvssD7qqKOWuwwAAGAF27p16z9399pd2+82weioo47K4uLicpcBAACsYFX1hd21m0oHAAAMTzACAACGN9dgVFUnV9WVVbW9qs7dzfqXV9Wl08+nq+rGafuDqupj0/bLq+rX51knAAAwtrk9Y1RVq5JsSvKEJEtJLqmqLd29bWef7v7tmf6/meSR08UvJfmp7v5mVd0rySen235xXvUCAMBKdMstt2RpaSk33XTTcpdyQFmzZk3WrVuXgw46aK/6z/PlCycm2d7dVyVJVW1OclqSbXvo/9QkL0iS7r55pv2eMeUPAIBBLS0t5d73vneOOuqoVNVyl3NA6O5cd911WVpayvr16/dqm3kGjiOSXD2zvDRt+x5V9aAk65O8e6btyKr6+HQff7S70aKqOruqFqtqcceOHfu0eAAAWAluuumm3P/+9xeK7oCqyv3vf/87NMq2UkZiNia5oLu/tbOhu6/u7kckeXCSZ1TVD++6UXef390L3b2wdu33vIocAADuFoSiO+6O/p3NMxhdk+TImeV107bd2ZjkjbtbMR0p+mSSn92n1QEAAEzNMxhdkmRDVa2vqoMzCT9bdu1UVQ9Nct8kH5ppW1dVPzD9ft8kP5PkyjnWCgAADGxuwai7b01yTpKLklyR5M3dfXlVnVdVp8503Zhkc3f3TNvDknykqi5L8r4kL+3uT8yrVgAAYO+88IUvzEtf+tJ9sq+zzjorF1xwwT7Z1101z7fSpbv/Kslf7dL2B7ssv3A3270rySPmWRsAAMBOcw1GAADAvvOit1+ebV/8yj7d58Mf+EN5wS/82Pft85KXvCSvec1rcvjhh+fII4/M8ccfn89+9rN59rOfnR07duSQQw7Jq171qjzgAQ/IIx7xiHzuc5/LPe5xj3z961/PQx/60Fx11VW3+/uELr744jz3uc/NrbfemhNOOCGveMUrcs973jPnnntutmzZktWrV+eJT3xiXvrSl+Ytb3lLXvSiF2XVqlW5z33uk/e///13+e9BMAIAAPZo69at2bx5cy699NLceuutOe6443L88cfn7LPPzitf+cps2LAhH/nIR/KsZz0r7373u3Psscfmfe97Xx772MfmwgsvzJOe9KTbDUU33XRTzjrrrFx88cU5+uij8/SnPz2veMUr8rSnPS1vfetb86lPfSpVlRtvvDFJct555+Wiiy7KEUcc8e22u0owAgCAA8TtjezMwwc+8IH80i/9Ug455JAkyamnnpqbbropf/d3f5fTTz/92/2++c1vJknOPPPMvOlNb8pjH/vYbN68Oc961rNu9xhXXnll1q9fn6OPPjpJ8oxnPCObNm3KOeeckzVr1uRXf/VXc8opp+SUU05Jkpx00kk566yzcsYZZ+TJT37yPjnPlfJ7jAAAgAPEbbfdlkMPPTSXXnrptz9XXHFFkklwesc73pHrr78+W7duzeMe97g7fZzVq1fnox/9aJ7ylKfkwgsvzMknn5wkeeUrX5kXv/jFufrqq3P88cfnuuuuu8vnJBgBAAB79KhHPSpve9vb8i//8i/56le/mre//e055JBDsn79+rzlLW9JknR3LrvssiTJve51r5xwwgl5znOek1NOOSWrVq263WM85CEPyec///ls3749SfK6170uj370o/O1r30tX/7yl/PzP//zefnLX/7tY3z2s5/NT/zET+S8887L2rVrc/XVV9/l8zSVDgAA2KPjjjsuZ555Zo455pgcfvjhOeGEE5Ikr3/96/Mbv/EbefGLX5xbbrklGzduzDHHHJNkMp3u9NNPz3vf+969OsaaNWvy53/+5zn99NO//fKFX//1X8/111+f0047LTfddFO6Oy972cuSJM973vPymc98Jt2dxz/+8d8+7l1R3/3rgw5cCwsLvbi4uNxlAADAPnXFFVfkYQ972HKXcUDa3d9dVW3t7oVd+5pKBwAADM9UOgAAYK6e/exn54Mf/OB3tT3nOc/JM5/5zGWq6HsJRgAAsMJ1d6pqucu40zZt2rTfj3lHHxkylQ4AAFawNWvW5LrrrrvD/9AfWXfnuuuuy5o1a/Z6GyNGAACwgq1bty5LS0vZsWPHcpdyQFmzZk3WrVu31/0FIwAAWMEOOuigrF+/frnLuNszlQ4AABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIY312BUVSdX1ZVVtb2qzt3N+pdX1aXTz6er6sZp+7FV9aGquryqPl5VZ86zTgAAYGyr57XjqlqVZFOSJyRZSnJJVW3p7m07+3T3b8/0/80kj5wufiPJ07v7M1X1wCRbq+qi7r5xXvUCAADjmueI0YlJtnf3Vd19c5LNSU77Pv2fmuSNSdLdn+7uz0y/fzHJtUnWzrFWAABgYPMMRkckuXpmeWna9j2q6kFJ1id5927WnZjk4CSfnUONAAAAK+blCxuTXNDd35ptrKoHJHldkmd29227blRVZ1fVYlUt7tixYz+VCgAA3N3MMxhdk+TImeV107bd2ZjpNLqdquqHkvx/SX6vuz+8u426+/zuXujuhbVrzbQDAADunHkGo0uSbKiq9VV1cCbhZ8uunarqoUnum+RDM20HJ3lrktd29wVzrBEAAGB+wai7b01yTpKLklyR5M3dfXlVnVdVp8503Zhkc3f3TNsZSR6V5KyZ13kfO69aAQCAsdV355ED18LCQi8uLi53GQAAwApWVVu7e2HX9pXy8gUAAIBlIxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhjfXYFRVJ1fVlVW1varO3c36l1fVpdPPp6vqxpl176iqG6vqwnnWCAAAsHpeO66qVUk2JXlCkqUkl1TVlu7etrNPd//2TP/fTPLImV38cZJDkvxv86oRAAAgme+I0YlJtnf3Vd19c5LNSU77Pv2fmuSNOxe6++IkX51jfQAAAEnmG4yOSHL1zPLStO17VNWDkqxP8u47coCqOruqFqtqcceOHXe6UAAAYGwr5eULG5Nc0N3fuiMbdff53b3Q3Qtr166dU2kAAMDd3TyD0TVJjpxZXjdt252NmZlGBwAAsD/NMxhdkmRDVa2vqoMzCT9bdu1UVQ9Nct8kH5pjLQAAAHs0t2DU3bcmOSfJRUmuSPLm7r68qs6rqlNnum5Msrm7e3b7qvpAkrckeXxVLVXVk+ZVKwAAMLbaJY8csBYWFnpxcXG5ywAAAFawqtra3Qu7tq+Uly8AAAAsG8EIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMLy5BqOqOrmqrqyq7VV17m7Wv7yqLp1+Pl1VN86se0ZVfWb6ecY86wQAAMa2el47rqpVSTYleUKSpSSXVNWW7t62s093//ZM/99M8sjp9/sleUGShSSdZOt02xvmVS8AADCueY4YnZhke3df1d03J9mc5LTv0/+pSd44/f6kJO/q7uunYehdSU6eY60AAMDA5hmMjkhy9czy0rTte1TVg5KsT/LuO7JtVZ1dVYtVtbhjx459UjQAADCelfLyhY1JLujub92Rjbr7/O5e6O6FtWvXzqk0AADg7m6eweiaJEfOLK+btu3OxnxnGt0d3RYAAOAumWcwuiTJhqpaX1UHZxJ+tuzaqaoemuS+ST4003xRkidW1X2r6r5JnjhtAwAA2Ofm9la67r61qs7JJNCsSvLq7r68qs5LstjdO0PSxiSbu7tntr2+qv5TJuEqSc7r7uvnVSsAADC2mskjB7SFhYVeXFxc7jIAAIAVrKq2dvfCru0r5eULAAAAy0YwAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHh7FYyq6jlV9UM18WdV9bGqeuK8iwMAANgf9nbE6N9391eSPDHJfZM8Lckfzq0qAACA/Whvg1FN//z5JK/r7stn2gAAAA5oexuMtlbVOzMJRhdV1b2T3Da/sgAAAPaf1XvZ71eTHJvkqu7+RlXdL8kz51YVAADAfrS3I0Y/leTK7r6xqn4lye8n+fL8ygIAANh/9nbE6BVJjqmqY5L8TpI/TfLaJI+eV2EHshe9/fJs++JXlrsMAABYdg9/4A/lBb/wY8tdxu3a2xGjW7u7k5yW5L9296Yk955fWQAAAPvP3o4YfbWqnp/Ja7p/tqrukeSg+ZV1YDsQEjEAAPAdeztidGaSb2by+4z+Mcm6JH88t6oAAAD2o70KRtMw9Pok96mqU5Lc1N2vnWtlAAAA+8leBaOqOiPJR5OcnuSMJB+pqqfMszAAAID9ZW+fMfq9JCd097VJUlVrk/xNkgvmVRgAAMD+srfPGN1jZyiauu4ObAsAALCi7e2I0Tuq6qIkb5wun5nkr+ZTEgAAwP61V8Gou59XVf8uyUnTpvO7+63zKwsAAGD/2dsRo3T3XyT5iznWAgAAsCy+bzCqqq8m6d2tStLd/UNzqQoAAGA/+r7BqLvvvb8KAQAAWC5zfbNcVZ1cVVdW1faqOncPfc6oqm1VdXlVvWGm/Y+q6pPTz5nzrBMAABjbXj9jdEdV1aokm5I8IclSkkuqakt3b5vpsyHJ85Oc1N03VNXh0/Z/m+S4JMcmuWeS91bVX3f3V+ZVLwAAMK55jhidmGR7d1/V3Tcn2ZzktF36/FqSTd19Q5LM/K6khyd5f3ff2t1fT/LxJCfPsVYAAGBg8wxGRyS5emZ5ado26+gkR1fVB6vqw1W1M/xcluTkqjqkqg5L8tgkR+56gKo6u6oWq2pxx44dczgFAABgBHObSncHjr8hyWOSrEvy/qr68e5+Z1WdkOTvkuxI8qEk39p14+4+P8n5SbKwsLC7t+cBAADcrnmOGF2T7x7lWTdtm7WUZEt339Ldn0vy6UyCUrr7Jd19bHc/IZPXg396jrUCAAADm2cwuiTJhqpaX1UHJ9mYZMsufd6WyWhRplPmjk5yVVWtqqr7T9sfkeQRSd45x1oBAICBzW0qXXffWlXnJLkoyaokr+7uy6vqvCSL3b1luu6JVbUtk6lyz+vu66pqTZIPVFWSfCXJr3T3rfOqFQAAGFt13z0ezVlYWOjFxcXlLgMAAFjBqmprdy/s2j7XX/AKAABwIBCMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMObazCqqpOr6sqq2l5V5+6hzxlVta2qLq+qN8y0/9/Ttiuq6r9UVc2zVgAAYFyr57XjqlqVZFOSJyRZSnJJVW3p7m0zfTYkeX6Sk7r7hqo6fNr+00lOSvKIade/TfLoJO+dV70AAMC45jlidGKS7d19VXffnGRzktN26fNrSTZ19w1J0t3XTts7yZokBye5Z5KDkvzTHGsFAAAGNs9gdESSq2eWl6Zts45OcnRVfbCqPlxVJydJd38oyXuSfGn6uai7r9j1AFV1dlUtVtXijh075nISAADA3d9yv3xhdZINSR6T5KlJXlVVh1bVg5M8LMm6TMLU46rqZ3fduLvP7+6F7l5Yu3btfiwbAAC4O5lnMLomyZEzy+umbbOWkmzp7lu6+3NJPp1JUPqlJB/u7q9199eS/HWSn5pjrQAAwMDmGYwuSbKhqtZX1cFJNibZskuft2UyWpSqOiyTqXVXJfmHJI+uqtVVdVAmL174nql0AAAA+8LcglF335rknCQXZRJq3tzdl1fVeVV16rTbRUmuq6ptmTxT9Lzuvi7JBUk+m+QTSS5Lcll3v31etQIAAGOr7l7uGvaJhYWFXlxcXO4yAACAFayqtnb3wq7ty/3yBQAAgGUnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGN9dgVFUnV9WVVbW9qs7dQ58zqmpbVV1eVW+Ytj22qi6d+dxUVb84z1oBAIBxrZ7XjqtqVZJNSZ6QZCnJJVW1pbu3zfTZkOT5SU7q7huq6vAk6e73JDl22ud+SbYneee8agUAAMY2zxGjE5Ns7+6ruvvmJJuTnLZLn19Lsqm7b0iS7r52N/t5SpK/7u5vzLFWAABgYPMMRkckuXpmeWnaNuvoJEdX1Qer6sNVdfJu9rMxyRt3d4CqOruqFqtqcceOHfukaAAAYDzL/fKF1Uk2JHlMkqcmeVVVHbpzZVU9IMmPJ7lodxt39/ndvdDdC2vXrp1/tQAAwN3SPIPRNUmOnFleN22btZRkS3ff0t2fS/LpTILSTmckeWt33zLHOgEAgMHNMxhdkmRDVa2vqoMzmRK3ZZc+b8tktChVdVgmU+uumln/1OxhGh0AAMC+Mrdg1N23Jjknk2lwVyR5c3dfXlXnVdWp024XJbmuqrYleU+S53X3dUlSVUdlMuL0vnnVCAAAkCTV3ctdwz6xsLDQi4uLy10GAACwglXV1u5e2LV9uV++AAAAsOwEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADA8wQgAABieYAQAAAxPMAIAAIYnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhCUYAAMDw5hqMqurkqrqyqrZX1bl76HNGVW2rqsur6g0z7T9SVe+sqium64+aZ60AAMC4Vs9rx1W1KsmmJE9IspTkkqra0t3bZvpsSPL8JCd19w1VdfjMLl6b5CXd/a6quleS2+ZVKwAAMLZ5jhidmGR7d1/V3Tcn2ZzktF36/FqSTd19Q5J097VJUlUPT7K6u981bf9ad39jjrUCAAADm2cwOiLJ1TPLS9O2WUcnObqqPlhVH66qk2fab6yqv6yqv6+qP56OQH2Xqjq7qharanHHjh1zOQkAAODub7lfvrA6yYYkj0ny1CSvqqpDp+0/m+S5SU5I8qNJztp14+4+v7sXunth7dq1+6lkAADg7maeweiaJEfOLK+bts1aSrKlu2/p7s8l+XQmQWkpyaXTaXi3JnlbkuPmWCsAADCweQajS5JsqKr1VXVwko1JtuzS522ZjBalqg7LZArdVdNtD62qncNAj0uyLQAAAHMwt2A0Hek5J8lFSa5I8ubuvryqzquqU6fdLkpyXVVtS/KeJM/r7uu6+1uZTKO7uKo+kaSSvGpetQIAAGOr7l7uGvaJhYWFXlxcXO4yAACAFayqtnb3wq7ty/3yBQAAgGUnGAEAAMMTjAAAgOEJRgAAwPAEIwAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYAQAAwxOMAACA4QlGAADA8AQjAABgeIIRAAAwPMEIAAAYnmAEAAAMTzACAACGJxgBAADDE4wAAIDhVXcvdw37RFXtSPKF5a5jxmFJ/nm5i+CA5NrhrnD9cGe5drgrXD/cWctx7Tyou9fu2ni3CUYrTVUtdvfCctfBgce1w13h+uHOcu1wV7h+uLNW0rVjKh0AADA8wQgAABieYDQ/5y93ARywXDvcFa4f7izXDneF64c7a8VcO54xAgAAhmfECAAAGJ5gBAAADE8w2seq6uSqurKqtlfVuctdDytbVR1ZVe+pqm1VdXlVPWfafr+qeldVfWb6532Xu1ZWpqpaVVV/X1UXTpfXV9VHpvegN1XVwctdIytTVR1aVRdU1aeq6oqq+in3HvZGVf329GfWJ6vqjVW1xr2HPamqV1fVtVX1yZm23d5rauK/TK+jj1fVcfuzVsFoH6qqVUk2Jfm5JA9P8tSqevjyVsUKd2uS3+nuhyf5ySTPnl4z5ya5uLs3JLl4ugy785wkV8ws/1GSl3f3g5PckORXl6UqDgT/T5J3dPdDkxyTyXXk3sP3VVVHJPnfkyx0979OsirJxrj3sGf/b5KTd2nb073m55JsmH7OTvKK/VRjEsFoXzsxyfbuvqq7b06yOclpy1wTK1h3f6m7Pzb9/tVM/mFyRCbXzWum3V6T5BeXpUBWtKpal+TfJvnT6XIleVySC6ZdXDvsVlXdJ8mjkvxZknT3zd19Y9x72Durk/xAVa1OckiSL8W9hz3o7vcnuX6X5j3da05L8tqe+HCSQ6vqAful0AhG+9oRSa6eWV6atsHtqqqjkjwyyUeS/HB3f2m66h+T/PBy1cWK9idJfjfJbdPl+ye5sbtvnS67B7En65PsSPLn06mYf1pVPxj3Hm5Hd1+T5KVJ/iGTQPTlJFvj3sMds6d7zbL+W1owghWgqu6V5C+S/FZ3f2V2XU/eqe+9+nyXqjolybXdvXW5a+GAtDrJcUle0d2PTPL17DJtzr2H3Zk+C3JaJuH6gUl+MN87TQr22kq61whG+9Y1SY6cWV43bYM9qqqDMglFr+/uv5w2/9POoePpn9cuV32sWCclObWqPp/JtN3HZfLMyKHT6S2JexB7tpRkqbs/Ml2+IJOg5N7D7fk3ST7X3Tu6+5Ykf5nJ/ci9hztiT/eaZf23tGC0b12SZMP0zSwHZ/Iw4pZlrokVbPpMyJ8luaK7XzazakuSZ0y/PyPJ/9zftbGydffzu3tddx+Vyb3m3d39y0nek+Qp026uHXaru/8xydVV9ZBp0+OTbIt7D7fvH5L8ZFUdMv0ZtvPace/hjtjTvWZLkqdP3073k0m+PDPlbu5qMnrFvlJVP5/JvP9VSV7d3S9Z3opYyarqZ5J8IMkn8p3nRP5jJs8ZvTnJjyT5QpIzunvXBxchSVJVj0ny3O4+pap+NJMRpPsl+fskv9Ld31zG8lihqurYTF7ccXCSq5I8M5P/MHXv4fuqqhclOTOTN6v+fZL/NZPnQNx7+B5V9cYkj0lyWJJ/SvKCJG/Lbu4107D9XzOZnvmNJM/s7sX9VqtgBAAAjM5UOgAAYHiCEQAAMDzBCAAAGJ5gBAAADE8wAgAAhicYATCsqnpMVV243HUAsPwEIwAAYHiCEQArXlX9SlV9tKourar/XlWrquprVfXyqrq8qi6uqrXTvsdW1Yer6uNV9daquu+0/cFV9TdVdVlVfayq/tV09/eqqguq6lNV9frpLxgEYDCCEQArWlU9LMmZSU7q7mOTfCvJLyf5wSSL3f1jSd6XyW9TT5LXJvkP3f2IJJ+YaX99kk3dfUySn07ypWn7I5P8VpKHJ/nRJCfN+ZQAWIFWL3cBAHA7Hp/k+CSXTAdzfiDJtUluS/KmaZ//keQvq+o+SQ7t7vdN21+T5C1Vde8kR3T3W5Oku29Kkun+PtrdS9PlS5McleRv535WAKwoghEAK10leU13P/+7Gqv+z1369Z3c/zdnvn8rfjYCDMlUOgBWuouTPKWqDk+SqrpfVT0ok59hT5n2+V+S/G13fznJDVX1s9P2pyV5X3d/NclSVf3idB/3rKpD9udJALCy+V8xAFa07t5WVb+f5J1VdY8ktyR5dpKvJzlxuu7aTJ5DSpJnJHnlNPhcleSZ0/anJfnvVXXedB+n78fTAGCFq+47O/MAAJZPVX2tu++13HUAcPdgKh0AADA8I0YAAMDwjBgBAADDE4wAAIDhCUYAAMDwBCMAAGB4ghEAADC8/x8jW2fzSZQUjQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1008x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(14, 7))\n",
    "plt.title(\"Loss VS Epoch\")\n",
    "\n",
    "plt.plot(dev_loss_by_epoch, label=\"dev_loss\")\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model\n",
    "torch.save(model, 'RoBERTa.pth')# epochs = 10, lr=1e-1 ->25:1e-3 ->50:1e-6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model = torch.load('RoBERTa.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_logits:  tensor([[1193.0547, 1193.0547]], device='cuda:0')\n",
      "y_pred:  tensor([0], device='cuda:0')\n",
      "label:  tensor([0], device='cuda:0')\n",
      "test_accuracy =  0.55\n"
     ]
    }
   ],
   "source": [
    "test_model.eval()\n",
    "num_correct_pred = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for i in range(0, test_raw_data.shape[0]):\n",
    "        prompt = test_raw_data.iloc[i]['question'] + \". \" + test_raw_data.iloc[i]['premise']\n",
    "        choice0 = test_raw_data.iloc[i]['choice1']\n",
    "        choice1 = test_raw_data.iloc[i]['choice2']\n",
    "        label = torch.tensor(test_raw_data.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "\n",
    "        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)\n",
    "        outputs = test_model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)\n",
    "\n",
    "        test_logits = outputs.logits\n",
    "\n",
    "        #calculate accuracy\n",
    "        y_pred = 1 if outputs.logits[0][1] > outputs.logits[0][0] else 0\n",
    "        y_pred = torch.tensor(y_pred).unsqueeze(0).to(device)\n",
    "\n",
    "        if y_pred == label:\n",
    "            print(\"test_logits: \", test_logits)\n",
    "            print(\"y_pred: \", y_pred)\n",
    "            print(\"label: \", label)\n",
    "            num_correct_pred += 1\n",
    "\n",
    "acc = num_correct_pred / test_raw_data.shape[0]\n",
    "print(\"test_accuracy = \", acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------Epoch: 1--------------\n",
      "Training for epoch 1.......\n",
      "train_loss:  tensor(11.3468, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[ 3.5860, 14.9327]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(0.7526, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[11.2423, 11.3579]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "av_train_loss:  3.2429466247558594\n",
      "--------------Epoch: 2--------------\n",
      "Training for epoch 2.......\n",
      "train_loss:  tensor(0., device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[12.2284, -8.9319]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(0., device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[13.1985, -6.4186]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "av_train_loss:  3.509030342102051\n",
      "--------------Epoch: 3--------------\n",
      "Training for epoch 3.......\n",
      "train_loss:  tensor(8.2847e-05, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[ 1.7571, -7.6411]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(10.3472, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[ 0.1527, 10.4999]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "av_train_loss:  3.6276304721832275\n",
      "--------------Epoch: 4--------------\n",
      "Training for epoch 4.......\n",
      "train_loss:  tensor(3.1881, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[-0.1453,  3.0006]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(10.0704, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[-15.3993,  -5.3290]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "av_train_loss:  4.204177379608154\n",
      "--------------Epoch: 5--------------\n",
      "Training for epoch 5.......\n",
      "train_loss:  tensor(0.0029, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[ 0.1989, -5.6282]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n",
      "train_loss:  tensor(12.4747, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "train_logits:  tensor([[-4.8200,  7.6547]], device='cuda:0', grad_fn=<ViewBackward0>)\n",
      "label:  tensor([0], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "# Revise\n",
    "# Training\n",
    "ce = nn.CrossEntropyLoss()\n",
    "softmax = nn.Softmax(dim=0)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "\n",
    "epochs = 10\n",
    "per_num_epoch = 1\n",
    "\n",
    "# train_acc = np.zeros(epochs)\n",
    "train_loss_by_epoch = np.zeros(epochs)\n",
    "dev_acc = np.zeros(epochs)\n",
    "dev_loss_by_epoch = np.zeros(epochs)\n",
    "\n",
    "start_time = datetime.now()\n",
    "\n",
    "for j in range(epochs):\n",
    "    if j % per_num_epoch == 0:\n",
    "        print('--------------Epoch: ' + str(j+1) + '--------------')\n",
    "    \n",
    "    if j % per_num_epoch == 0:\n",
    "        print(f'Training for epoch {j + 1}.......')\n",
    "    av_train_loss = 0\n",
    "    # print(\"av_train_loss_original: \", av_train_loss)\n",
    "    model.train()\n",
    "    for i in range(0, train_raw_data.shape[0]):\n",
    "        # print(\"av_train_loss_track: \", av_train_loss)\n",
    "        prompt = train_raw_data.iloc[i]['question'] + \". \" + train_raw_data.iloc[i]['premise']\n",
    "        choice0 = train_raw_data.iloc[i]['choice1']\n",
    "        choice1 = train_raw_data.iloc[i]['choice2']\n",
    "        label = torch.tensor(train_raw_data.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "        # print(\"label is: \", label)\n",
    "        # label = torch.tensor(rawdata.iloc[i]['label']).unsqueeze(0).to(device)\n",
    "\n",
    "        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)\n",
    "        # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=label)\n",
    "        \n",
    "        train_loss = outputs.loss\n",
    "        train_logits = outputs.logits\n",
    "        av_train_loss += train_loss\n",
    "\n",
    "        if i == 0:\n",
    "            print(\"train_loss: \", train_loss)\n",
    "            print(\"train_logits: \", train_logits)\n",
    "            print(\"label: \", label)\n",
    "        if i == 1:\n",
    "            print(\"train_loss: \", train_loss)\n",
    "            print(\"train_logits: \", train_logits)\n",
    "            print(\"label: \", label)\n",
    "\n",
    "        train_loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        # learning rate decay\n",
    "        # if j == 25:\n",
    "        #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
    "        # elif j == 50:\n",
    "        #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-7)\n",
    "        \n",
    "        # \n",
    "\n",
    "    train_loss_by_epoch[j] = av_train_loss / train_raw_data.shape[0]\n",
    "    print(\"av_train_loss: \", train_loss_by_epoch[j])\n",
    "\n",
    "end_time = datetime.now()\n",
    "print(f'Training completed in {str(end_time - start_time)}')\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9a411169b02b4f5d985b282c4f140db7a58e4cacae7bbcf8f29fd937be3ae09c"
  },
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit ('NLP_env': venv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}