FLtest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import transformers\n",
    "import matplotlib.pyplot as plt\n",
    "from transformers import AutoTokenizer, AutoModel, AdamW, get_linear_schedule_with_warmup, BertTokenizer, BertModel\n",
    "from transformers import Trainer, TrainingArguments\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import torch.nn as nn\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from torch.nn.utils import clip_grad_norm\n",
    "import gc\n",
    "from transformers.adapters import AutoAdapterModel, RobertaAdapterModel\n",
    "from transformers import RobertaTokenizer\n",
    "from datasets import load_dataset, Dataset\n",
    "from transformers import TrainingArguments, AdapterTrainer, EvalPrediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = pd.read_csv(\"sar_and_meta_train.csv\")[:20000]\n",
    "test_fl = pd.read_csv(\"sar_and_meta_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_size=0.1\n",
    "seed=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl, val_fl = train_test_split(train_fl, test_size=val_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = Dataset.from_pandas(train_fl, preserve_index=False)\n",
    "val_fl = Dataset.from_pandas(val_fl, preserve_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
      "loading configuration file config.json from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"vinai/bertweet-base\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 130,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"tokenizer_class\": \"BertweetTokenizer\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64001\n",
      "}\n",
      "\n",
      "loading file vocab.txt from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/vocab.txt\n",
      "loading file bpe.codes from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/bpe.codes\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at None\n",
      "loading configuration file config.json from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"vinai/bertweet-base\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 130,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"tokenizer_class\": \"BertweetTokenizer\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64001\n",
      "}\n",
      "\n",
      "emoji is not installed, thus not converting emoticons or emojis into text. Install emoji: pip3 install emoji==0.6.0\n",
      "Adding <mask> to the vocabulary\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cda0339c39a141ff8099e73fb243173d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/18000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1aee8663f28b4a7fbb00f2b7a45d9864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def encode_batch(batch):\n",
    "  \"\"\"Encodes a batch of input data using the model tokenizer.\"\"\"\n",
    "  return tokenizer(batch[\"text\"], add_special_tokens=True, padding='max_length', max_length=128, truncation=True, return_attention_mask=True)\n",
    "\n",
    "# Encode the input data\n",
    "train_fl = train_fl.map(encode_batch, batched=True)\n",
    "val_fl = val_fl.map(encode_batch, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fl = train_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "train_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])\n",
    "\n",
    "val_fl = val_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "val_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_p = torch.load(\"checkpoints/epoch4@sid32726483.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 130,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"tokenizer_class\": \"BertweetTokenizer\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64001\n",
      "}\n",
      "\n",
      "loading weights file pytorch_model.bin from cache at /home/xl2473/.cache/huggingface/hub/models--vinai--bertweet-base/snapshots/118ab1d567653bec16bbb081eafb6f8942f72108/pytorch_model.bin\n",
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"transformers_version\": \"4.26.1\"\n",
      "}\n",
      "\n",
      "Some weights of the model checkpoint at vinai/bertweet-base were not used when initializing RobertaModelWithHeads: ['lm_head.bias', 'lm_head.decoder.weight', 'lm_head.decoder.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias']\n",
      "- This IS expected if you are initializing RobertaModelWithHeads from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModelWithHeads from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of RobertaModelWithHeads were initialized from the model checkpoint at vinai/bertweet-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaModelWithHeads for predictions without further training.\n",
      "Generation config file not found, using a generation config created from the model config.\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaConfig, RobertaModelWithHeads\n",
    "\n",
    "config = RobertaConfig.from_pretrained(\n",
    "    \"vinai/bertweet-base\",\n",
    "    num_labels=3,\n",
    ")\n",
    "model = RobertaModelWithHeads.from_pretrained(\n",
    "    \"vinai/bertweet-base\",\n",
    "    config=config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.roberta.load_state_dict(model_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Adding adapter 'fladapter'.\n",
      "Adding head 'fladapter' with config {'head_type': 'classification', 'num_labels': 3, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'none': 0, 'sarcasm': 1, 'metaphor': 2}, 'use_pooler': False, 'bias': True}.\n"
     ]
    }
   ],
   "source": [
    "# Add a new adapter\n",
    "model.add_adapter(\"fladapter\")\n",
    "# Add a matching classification head\n",
    "model.add_classification_head(\n",
    "    \"fladapter\",\n",
    "    num_labels=3,\n",
    "    id2label={ 0: \"none\", 1: \"sarcasm\", 2:\"metaphor\"}\n",
    "  )\n",
    "# Activate the adapter\n",
    "model.train_adapter(\"fladapter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    learning_rate=1e-4,\n",
    "    num_train_epochs=6,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    logging_steps=200,\n",
    "    output_dir=\"./training_output\",\n",
    "    overwrite_output_dir=True,\n",
    "    # The next line is important to ensure the dataset labels are properly passed to the model\n",
    "    remove_unused_columns=False,\n",
    ")\n",
    "\n",
    "def compute_accuracy(p: EvalPrediction):\n",
    "  preds = np.argmax(p.predictions, axis=1)\n",
    "  return {\"acc\": (preds == p.label_ids).mean()}\n",
    "\n",
    "trainer = AdapterTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_fl,\n",
    "    eval_dataset=val_fl,\n",
    "    compute_metrics=compute_accuracy,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 18000\n",
      "  Num Epochs = 6\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 3378\n",
      "  Number of trainable parameters = 1487427\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3378' max='3378' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3378/3378 10:23, Epoch 6/6]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.778300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.773100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.763800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.764100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.769500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.770800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.767500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.761200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.763200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.768000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>0.755000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>0.774700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>0.757300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>0.767700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.760400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>0.765300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./training_output/checkpoint-500\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-1000\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-1000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1000/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-1500\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-1500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-1500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-2000\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-2000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2000/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-2500\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-2500/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-2500/fladapter/pytorch_model_head.bin\n",
      "Saving model checkpoint to ./training_output/checkpoint-3000\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/adapter_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/checkpoint-3000/fladapter/head_config.json\n",
      "Module weights saved in ./training_output/checkpoint-3000/fladapter/pytorch_model_head.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3378, training_loss=0.7662090755482973, metrics={'train_runtime': 623.2928, 'train_samples_per_second': 173.273, 'train_steps_per_second': 5.42, 'total_flos': 7227244071936000.0, 'train_loss': 0.7662090755482973, 'epoch': 6.0})"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 2000\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.7545523643493652,\n",
       " 'eval_acc': 0.566,\n",
       " 'eval_runtime': 5.3778,\n",
       " 'eval_samples_per_second': 371.897,\n",
       " 'eval_steps_per_second': 11.715,\n",
       " 'epoch': 6.0}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fl = Dataset.from_pandas(test_fl, preserve_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71bc8c57c21843da9c82bb14529ca86e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/18210 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_fl = test_fl.map(encode_batch, batched=True)\n",
    "test_fl = test_fl.rename_column(\"label\", \"labels\")\n",
    "# Transform to pytorch tensors and only output the required columns\n",
    "test_fl.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 18210\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PredictionOutput(predictions=array([[ 1.2435882 ,  0.9313903 , -2.2752206 ],\n",
       "       [ 1.2435883 ,  0.9313903 , -2.2752206 ],\n",
       "       [ 1.2435883 ,  0.9313903 , -2.2752204 ],\n",
       "       ...,\n",
       "       [ 1.2435883 ,  0.9313903 , -2.2752204 ],\n",
       "       [ 1.2435883 ,  0.9313903 , -2.2752204 ],\n",
       "       [ 1.2435883 ,  0.93139017, -2.2752204 ]], dtype=float32), label_ids=array([1, 0, 0, ..., 0, 0, 1]), metrics={'test_loss': 0.770438015460968, 'test_acc': 0.5616694124107633, 'test_runtime': 49.1647, 'test_samples_per_second': 370.388, 'test_steps_per_second': 11.594})"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = trainer.predict(test_fl)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}