selftest2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import transformers\n",
    "import matplotlib.pyplot as plt\n",
    "from transformers import AutoTokenizer, AutoModel, AdamW, get_linear_schedule_with_warmup, BertTokenizer, BertModel\n",
    "from transformers import Trainer, TrainingArguments\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import torch.nn as nn\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from torch.nn.utils import clip_grad_norm\n",
    "import gc\n",
    "from transformers.adapters import AutoAdapterModel, RobertaAdapterModel\n",
    "from transformers import RobertaTokenizer\n",
    "from datasets import load_dataset, Dataset\n",
    "from transformers import TrainingArguments, AdapterTrainer, EvalPrediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = pd.read_csv(\"sar_and_meta_train.csv\")[:20000]\n",
    "test_fl = pd.read_csv(\"sar_and_meta_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>P: Or they think what are the odds it'll happe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Instead of having the uperclass already paying...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>P: Nope. She killed Ronnie twice. But then aga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>P: Yeah, Jon is Dany's brother's son; C: So th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Poverty engenders a social environment charact...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>1</td>\n",
       "      <td>P: Bavaria is like Germany's Texas. If they do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>1</td>\n",
       "      <td>P: Dude, that's what breasts are for. Why else...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>0</td>\n",
       "      <td>P: then give it you cheap fuck; C: Why on eart...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>1</td>\n",
       "      <td>P: I wish they did this to TF2, that would be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>1</td>\n",
       "      <td>P: What I found surprising was how long it too...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                                               text\n",
       "0          1  P: Or they think what are the odds it'll happe...\n",
       "1          0  Instead of having the uperclass already paying...\n",
       "2          0  P: Nope. She killed Ronnie twice. But then aga...\n",
       "3          0  P: Yeah, Jon is Dany's brother's son; C: So th...\n",
       "4          0  Poverty engenders a social environment charact...\n",
       "...      ...                                                ...\n",
       "19995      1  P: Bavaria is like Germany's Texas. If they do...\n",
       "19996      1  P: Dude, that's what breasts are for. Why else...\n",
       "19997      0  P: then give it you cheap fuck; C: Why on eart...\n",
       "19998      1  P: I wish they did this to TF2, that would be ...\n",
       "19999      1  P: What I found surprising was how long it too...\n",
       "\n",
       "[20000 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_size=0.1\n",
    "seed=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl, val_fl = train_test_split(train_fl, test_size=val_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16270</th>\n",
       "      <td>0</td>\n",
       "      <td>Civilian Gun Control is a distraction from the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1383</th>\n",
       "      <td>1</td>\n",
       "      <td>P: Geez, did you get a souvenir photo with tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3472</th>\n",
       "      <td>0</td>\n",
       "      <td>P: Something tells me they were using abnormal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19900</th>\n",
       "      <td>0</td>\n",
       "      <td>As far as I'm concerned, the NRA showed itself...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2150</th>\n",
       "      <td>0</td>\n",
       "      <td>P: \"Look at this, this is my cocaine fort. No ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11284</th>\n",
       "      <td>1</td>\n",
       "      <td>P: As a brit - 10/10 would queue again; C: sti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11964</th>\n",
       "      <td>1</td>\n",
       "      <td>P: my car is worth less than my 980ti; C: When...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5390</th>\n",
       "      <td>0</td>\n",
       "      <td>You claim you support gun ownership, and reaso...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>1</td>\n",
       "      <td>P: Don't vote.; C: No, obviously you have to u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15795</th>\n",
       "      <td>0</td>\n",
       "      <td>How much would Obama have to raise taxes to co...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                                               text\n",
       "16270      0  Civilian Gun Control is a distraction from the...\n",
       "1383       1  P: Geez, did you get a souvenir photo with tha...\n",
       "3472       0  P: Something tells me they were using abnormal...\n",
       "19900      0  As far as I'm concerned, the NRA showed itself...\n",
       "2150       0  P: \"Look at this, this is my cocaine fort. No ...\n",
       "...      ...                                                ...\n",
       "11284      1  P: As a brit - 10/10 would queue again; C: sti...\n",
       "11964      1  P: my car is worth less than my 980ti; C: When...\n",
       "5390       0  You claim you support gun ownership, and reaso...\n",
       "860        1  P: Don't vote.; C: No, obviously you have to u...\n",
       "15795      0  How much would Obama have to raise taxes to co...\n",
       "\n",
       "[18000 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = Dataset.from_pandas(train_fl, preserve_index=False)\n",
    "val_fl = Dataset.from_pandas(val_fl, preserve_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'label': 0,\n",
       " 'text': 'Civilian Gun Control is a distraction from the real issue.'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_fl[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'text'],\n",
       "    num_rows: 2000\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "emoji is not installed, thus not converting emoticons or emojis into text. Install emoji: pip3 install emoji==0.6.0\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "000e11f19bff4c9bbe592d7e1c6c3220",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/18000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "172c3a7b675e4c0899b83a8eea8fa429",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def encode_batch(batch):\n",
    "  \"\"\"Encodes a batch of input data using the model tokenizer.\"\"\"\n",
    "  return tokenizer(batch[\"text\"], add_special_tokens=True, padding='max_length', max_length=128, truncation=True, return_attention_mask=True)\n",
    "\n",
    "# Encode the input data\n",
    "train_fl = train_fl.map(encode_batch, batched=True)\n",
    "val_fl = val_fl.map(encode_batch, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 18000\n",
       "})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 2000\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = train_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "train_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])\n",
    "\n",
    "val_fl = val_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "val_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 18000\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 2000\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xl2473/.local/lib/python3.8/site-packages/transformers/adapters/models/roberta/adapter_model.py:274: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "/home/xl2473/.local/lib/python3.8/site-packages/transformers/adapters/models/roberta/adapter_model.py:252: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at vinai/bertweet-base were not used when initializing RobertaModelWithHeads: ['lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.decoder.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias']\n",
      "- This IS expected if you are initializing RobertaModelWithHeads from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModelWithHeads from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaConfig, RobertaModelWithHeads\n",
    "\n",
    "config = RobertaConfig.from_pretrained(\n",
    "    \"vinai/bertweet-base\",\n",
    "    num_labels=3,\n",
    ")\n",
    "model = RobertaModelWithHeads.from_pretrained(\n",
    "    \"vinai/bertweet-base\",\n",
    "    config=config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a new adapter\n",
    "model.add_adapter(\"fladapter\")\n",
    "# Add a matching classification head\n",
    "model.add_classification_head(\n",
    "    \"fladapter\",\n",
    "    num_labels=3,\n",
    "    id2label={ 0: \"none\", 1: \"sarcasm\", 2:\"metaphor\"}\n",
    "  )\n",
    "# Activate the adapter\n",
    "model.train_adapter(\"fladapter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    learning_rate=1e-4,\n",
    "    num_train_epochs=6,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    logging_steps=200,\n",
    "    output_dir=\"./training_output\",\n",
    "    overwrite_output_dir=True,\n",
    "    # The next line is important to ensure the dataset labels are properly passed to the model\n",
    "    remove_unused_columns=False,\n",
    ")\n",
    "\n",
    "def compute_accuracy(p: EvalPrediction):\n",
    "  preds = np.argmax(p.predictions, axis=1)\n",
    "  return {\"acc\": (preds == p.label_ids).mean()}\n",
    "\n",
    "trainer = AdapterTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_fl,\n",
    "    eval_dataset=val_fl,\n",
    "    compute_metrics=compute_accuracy,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xl2473/.local/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 18000\n",
      "  Num Epochs = 6\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 3378\n",
      "  Number of trainable parameters = 1487427\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3378' max='3378' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3378/3378 10:19, Epoch 6/6]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.705800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.597300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.557700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.539500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.534900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.510500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.508300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.498000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.492100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.483300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>0.485000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>0.481400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>0.454800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>0.482100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.449200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>0.466300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./training_output/checkpoint-500\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-1000\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-1500\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-2000\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-2500\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-3000\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_model_head.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3378, training_loss=0.5126851286811897, metrics={'train_runtime': 625.046, 'train_samples_per_second': 172.787, 'train_steps_per_second': 5.404, 'total_flos': 7227244071936000.0, 'train_loss': 0.5126851286811897, 'epoch': 6.0})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 2000\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.4898378849029541,\n",
       " 'eval_acc': 0.768,\n",
       " 'eval_runtime': 5.6832,\n",
       " 'eval_samples_per_second': 351.911,\n",
       " 'eval_steps_per_second': 11.085,\n",
       " 'epoch': 6.0}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fl = Dataset.from_pandas(test_fl, preserve_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'text'],\n",
       "    num_rows: 18210\n",
       "})"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ffad85ddd004852889b97dbe48d1578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/18210 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_fl = test_fl.map(encode_batch, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fl = test_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "test_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 18210\n",
       "})"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 18210\n",
      "  Batch size = 32\n"
     ]
    }
   ],
   "source": [
    "res = trainer.predict(test_fl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PredictionOutput(predictions=array([[ 2.1516526 ,  3.1810842 , -6.1749334 ],\n",
       "       [ 3.612661  , -2.083609  , -1.1310233 ],\n",
       "       [ 3.5071766 , -3.184903  ,  0.616519  ],\n",
       "       ...,\n",
       "       [ 3.880916  , -2.956007  , -0.12211605],\n",
       "       [ 3.1370227 , -3.9134932 ,  1.7422873 ],\n",
       "       [ 0.8642302 ,  4.038598  , -6.1159067 ]], dtype=float32), label_ids=array([1, 0, 0, ..., 0, 0, 1]), metrics={'test_loss': 0.5104750990867615, 'test_acc': 0.7604612850082373, 'test_runtime': 47.9079, 'test_samples_per_second': 380.104, 'test_steps_per_second': 11.898})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}