nerTransformer4L.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cdcc97f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniconda/base/envs/kerasTF/lib/python3.10/site-packages/h5py/__init__.py:36: UserWarning: h5py is running against HDF5 1.14.3 when it was built against 1.14.2, this may cause problems\n",
      "  _warn((\"h5py is running against HDF5 {0} when it was built against {1}, \"\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "\n",
    "import os\n",
    "import keras\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras import layers\n",
    "from datasets import load_dataset\n",
    "from collections import Counter\n",
    "from conlleval import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d4db7895-8b71-4190-9ee4-7151c4091fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will initialize The Transformer model with just one block - 64 dim 6 attention heads\n",
    "# Total params: 1,401,098"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5149f955",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerBlock(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
    "        super().__init__()\n",
    "        self.att = keras.layers.MultiHeadAttention(\n",
    "            num_heads=num_heads, key_dim=embed_dim\n",
    "        )\n",
    "        self.ffn = keras.Sequential(\n",
    "            [\n",
    "                keras.layers.Dense(ff_dim, activation=\"relu\"),\n",
    "                keras.layers.Dense(embed_dim),\n",
    "            ]\n",
    "        )\n",
    "        self.layernorm1 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        #self.layernorm3 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        #self.layernorm4 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = keras.layers.Dropout(rate)\n",
    "        self.dropout2 = keras.layers.Dropout(rate)\n",
    "        #self.dropout3 = keras.layers.Dropout(rate)\n",
    "        #self.dropout4 = keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, inputs, training=False):\n",
    "        attn_output = self.att(inputs, inputs)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(inputs + attn_output)\n",
    "        ffn_output1 = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output1, training=training)\n",
    "        return self.layernorm2(out1 + ffn_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8aa45bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenAndPositionEmbedding(layers.Layer):\n",
    "    def __init__(self, maxlen, vocab_size, embed_dim):\n",
    "        super().__init__()\n",
    "        self.token_emb = keras.layers.Embedding(\n",
    "            input_dim=vocab_size, output_dim=embed_dim\n",
    "        )\n",
    "        self.pos_emb = keras.layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        maxlen = tf.shape(inputs)[-1]\n",
    "        positions = tf.range(start=0, limit=maxlen, delta=1)\n",
    "        position_embeddings = self.pos_emb(positions)\n",
    "        token_embeddings = self.token_emb(inputs)\n",
    "        return token_embeddings + position_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8c97228e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NERModel(keras.Model):\n",
    "    def __init__(\n",
    "        self, num_tags, vocab_size, maxlen=128, embed_dim=32, num_heads=2, ff_dim=32\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n",
    "        self.transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\n",
    "        self.dropout1 = layers.Dropout(0.1)\n",
    "        self.ff = layers.Dense(ff_dim, activation=\"relu\")\n",
    "        self.dropout2 = layers.Dropout(0.1)\n",
    "        self.ff_final = layers.Dense(num_tags, activation=\"softmax\")\n",
    "\n",
    "    def call(self, inputs, training=False):\n",
    "        x = self.embedding_layer(inputs)\n",
    "        x = self.transformer_block(x)\n",
    "        x = self.dropout1(x, training=training)\n",
    "        x = self.ff(x)\n",
    "        x = self.dropout2(x, training=training)\n",
    "        x = self.ff_final(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "12febd82",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"REQUESTS_CA_BUNDLE\"]=\"/Users/smatuk1/MSPref/Transformers/hfg.pem\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2322dec0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the module from /Users/smatuk1/.cache/huggingface/modules/datasets_modules/datasets/conll2003/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98 (last modified on Sun Dec  3 17:57:49 2023) since it couldn't be found locally at conll2003, or remotely on the Hugging Face Hub.\n"
     ]
    }
   ],
   "source": [
    "conll_data = load_dataset(\"conll2003\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6dd63b39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
      "        num_rows: 14041\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
      "        num_rows: 3250\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
      "        num_rows: 3453\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(conll_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b2e62115",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conll_data.save_to_disk(\"/Users/XXXXX/MSPref/Transformers/LLM/conll_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0a1b04c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_to_file(export_file_path, data):\n",
    "    with open(export_file_path, \"w\") as f:\n",
    "        for record in data:\n",
    "            ner_tags = record[\"ner_tags\"]\n",
    "            tokens = record[\"tokens\"]\n",
    "            if len(tokens) > 0:\n",
    "                f.write(\n",
    "                    str(len(tokens))\n",
    "                    + \"\\t\"\n",
    "                    + \"\\t\".join(tokens)\n",
    "                    + \"\\t\"\n",
    "                    + \"\\t\".join(map(str, ner_tags))\n",
    "                    + \"\\n\"\n",
    "                )\n",
    "\n",
    "\n",
    "#os.mkdir(\"conlldata\")\n",
    "#export_to_file(\"./conlldata/conll_train.txt\", conll_data[\"train\"])\n",
    "#export_to_file(\"./conlldata/conll_val.txt\", conll_data[\"validation\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "491fa29f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: '[PAD]', 1: 'O', 2: 'B-PER', 3: 'I-PER', 4: 'B-ORG', 5: 'I-ORG', 6: 'B-LOC', 7: 'I-LOC', 8: 'B-MISC', 9: 'I-MISC'}\n"
     ]
    }
   ],
   "source": [
    "def make_tag_lookup_table():\n",
    "    iob_labels = [\"B\", \"I\"]\n",
    "    ner_labels = [\"PER\", \"ORG\", \"LOC\", \"MISC\"]\n",
    "    all_labels = [(label1, label2) for label2 in ner_labels for label1 in iob_labels]\n",
    "    all_labels = [\"-\".join([a, b]) for a, b in all_labels]\n",
    "    all_labels = [\"[PAD]\", \"O\"] + all_labels\n",
    "    return dict(zip(range(0, len(all_labels) + 1), all_labels))\n",
    "\n",
    "\n",
    "mapping = make_tag_lookup_table()\n",
    "print(mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2ffd5579-cf98-46a2-9c99-617776486a0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vocabulary size restricted to 20K from https://huggingface.co/datasets/tner/conll2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "935cd53e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21009\n"
     ]
    }
   ],
   "source": [
    "all_tokens = sum(conll_data[\"train\"][\"tokens\"], [])\n",
    "all_tokens_array = np.array(list(map(str.lower, all_tokens)))\n",
    "\n",
    "counter = Counter(all_tokens_array)\n",
    "print(len(counter))\n",
    "\n",
    "num_tags = len(mapping)\n",
    "vocab_size = 20000\n",
    "\n",
    "# We only take (vocab_size - 2) most commons words from the training data since\n",
    "# the `StringLookup` class uses 2 additional tokens - one denoting an unknown\n",
    "# token and another one denoting a masking token\n",
    "vocabulary = [token for token, count in counter.most_common(vocab_size - 2)]\n",
    "\n",
    "# The StringLook class will convert tokens to token IDs\n",
    "lookup_layer = keras.layers.StringLookup(vocabulary=vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "156c1cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = tf.data.TextLineDataset(\"./conlldata/conll_train.txt\")\n",
    "val_data = tf.data.TextLineDataset(\"./conlldata/conll_val.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "deac8e4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'9\\tEU\\trejects\\tGerman\\tcall\\tto\\tboycott\\tBritish\\tlamb\\t.\\t3\\t0\\t7\\t0\\t0\\t0\\t7\\t0\\t0']\n"
     ]
    }
   ],
   "source": [
    "print(list(train_data.take(1).as_numpy_iterator()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eada8650",
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_record_to_training_data(record):\n",
    "    record = tf.strings.split(record, sep=\"\\t\")\n",
    "    length = tf.strings.to_number(record[0], out_type=tf.int32)\n",
    "    tokens = record[1 : length + 1]\n",
    "    tags = record[length + 1 :]\n",
    "    tags = tf.strings.to_number(tags, out_type=tf.int64)\n",
    "    tags += 1\n",
    "    return tokens, tags\n",
    "\n",
    "\n",
    "def lowercase_and_convert_to_ids(tokens):\n",
    "    tokens = tf.strings.lower(tokens)\n",
    "    return lookup_layer(tokens)\n",
    "\n",
    "\n",
    "# We use `padded_batch` here because each record in the dataset has a\n",
    "# different length.\n",
    "batch_size = 32\n",
    "train_dataset = (\n",
    "    train_data.map(map_record_to_training_data)\n",
    "    .map(lambda x, y: (lowercase_and_convert_to_ids(x), y))\n",
    "    .padded_batch(batch_size)\n",
    ")\n",
    "val_dataset = (\n",
    "    val_data.map(map_record_to_training_data)\n",
    "    .map(lambda x, y: (lowercase_and_convert_to_ids(x), y))\n",
    "    .padded_batch(batch_size)\n",
    ")\n",
    "\n",
    "#ner_model = NERModel(num_tags, vocab_size, embed_dim=32, num_heads=4, ff_dim=64)\n",
    "#NER model with embedings 64 dim 6 attention heads\n",
    "ner_model = NERModel(num_tags, vocab_size, embed_dim=64, num_heads=6, ff_dim=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "79e52ba9",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomNonPaddingTokenLoss(keras.losses.Loss):\n",
    "    def __init__(self, name=\"custom_ner_loss\"):\n",
    "        super().__init__(name=name)\n",
    "\n",
    "    def call(self, y_true, y_pred):\n",
    "        loss_fn = keras.losses.SparseCategoricalCrossentropy(\n",
    "            from_logits=False, reduction=tf.keras.losses.Reduction.NONE\n",
    "        )\n",
    "        loss = loss_fn(y_true, y_pred)\n",
    "        mask = tf.cast((y_true > 0), dtype=tf.float32)\n",
    "        loss = loss * mask\n",
    "        return tf.reduce_sum(loss) / tf.reduce_sum(mask)\n",
    "\n",
    "\n",
    "loss = CustomNonPaddingTokenLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e6bf2ab-6b71-46a0-b5d0-0c3c03ca729a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 20 epochs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fea61de8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "439/439 [==============================] - 10s 20ms/step - loss: 0.5547\n",
      "Epoch 2/20\n",
      "439/439 [==============================] - 11s 24ms/step - loss: 0.2296\n",
      "Epoch 3/20\n",
      "439/439 [==============================] - 10s 23ms/step - loss: 0.1362\n",
      "Epoch 4/20\n",
      "439/439 [==============================] - 11s 25ms/step - loss: 0.1023\n",
      "Epoch 5/20\n",
      "439/439 [==============================] - 11s 25ms/step - loss: 0.0796\n",
      "Epoch 6/20\n",
      "439/439 [==============================] - 11s 24ms/step - loss: 0.0665\n",
      "Epoch 7/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0600\n",
      "Epoch 8/20\n",
      "439/439 [==============================] - 10s 24ms/step - loss: 0.0503\n",
      "Epoch 9/20\n",
      "439/439 [==============================] - 11s 25ms/step - loss: 0.0409\n",
      "Epoch 10/20\n",
      "439/439 [==============================] - 10s 23ms/step - loss: 0.0339\n",
      "Epoch 11/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0308\n",
      "Epoch 12/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0286\n",
      "Epoch 13/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0267\n",
      "Epoch 14/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0239\n",
      "Epoch 15/20\n",
      "439/439 [==============================] - 12s 26ms/step - loss: 0.0232\n",
      "Epoch 16/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0213\n",
      "Epoch 17/20\n",
      "439/439 [==============================] - 12s 26ms/step - loss: 0.0200\n",
      "Epoch 18/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0243\n",
      "Epoch 19/20\n",
      "439/439 [==============================] - 12s 27ms/step - loss: 0.0179\n",
      "Epoch 20/20\n",
      "439/439 [==============================] - 11s 26ms/step - loss: 0.0149\n",
      "Model: \"ner_model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " token_and_position_embeddi  multiple                  1288192   \n",
      " ng (TokenAndPositionEmbedd                                      \n",
      " ing)                                                            \n",
      "                                                                 \n",
      " transformer_block (Transfo  multiple                  108096    \n",
      " rmerBlock)                                                      \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         multiple                  0         \n",
      "                                                                 \n",
      " dense_2 (Dense)             multiple                  4160      \n",
      "                                                                 \n",
      " dropout_3 (Dropout)         multiple                  0         \n",
      "                                                                 \n",
      " dense_3 (Dense)             multiple                  650       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1401098 (5.34 MB)\n",
      "Trainable params: 1401098 (5.34 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n",
      "None\n",
      "tf.Tensor([[  988 10950   204   628     6  3938   215  5773]], shape=(1, 8), dtype=int64)\n",
      "1/1 [==============================] - 0s 93ms/step\n",
      "[[[9.01800715e-13 1.37955965e-02 2.37635868e-05 3.30383121e-09\n",
      "   9.86180246e-01 2.93944833e-08 1.37163838e-07 6.14762186e-10\n",
      "   2.02552087e-07 1.06531406e-09]\n",
      "  [1.19094537e-15 1.00000000e+00 6.91733892e-10 7.00154171e-12\n",
      "   4.30191133e-10 1.26427668e-09 1.99155914e-10 7.16916457e-12\n",
      "   8.80925055e-11 2.52990406e-10]\n",
      "  [6.89982533e-12 2.99277121e-06 6.36628243e-08 5.83627702e-10\n",
      "   8.09865419e-09 6.92558046e-07 1.67403527e-07 2.95619793e-06\n",
      "   9.99956965e-01 3.60653503e-05]\n",
      "  [8.52667834e-15 1.00000000e+00 1.51206203e-09 2.47137893e-10\n",
      "   2.57890287e-09 1.58180011e-10 2.87846552e-10 6.99294702e-12\n",
      "   5.77218384e-10 8.75818081e-12]\n",
      "  [2.44995238e-15 1.00000000e+00 3.31883479e-11 6.63061495e-10\n",
      "   1.70336675e-10 3.61751823e-10 3.18461993e-11 1.23908744e-10\n",
      "   2.13855128e-10 5.83097515e-10]\n",
      "  [1.31626209e-14 1.00000000e+00 1.44862344e-10 5.14874310e-10\n",
      "   1.28072253e-09 2.12225307e-10 9.51935530e-10 1.23421134e-11\n",
      "   3.73382658e-09 4.76130697e-11]\n",
      "  [3.97138838e-13 1.26865825e-05 1.60973688e-08 8.36662510e-12\n",
      "   4.68655884e-07 1.55607208e-10 1.20622724e-06 5.41577272e-09\n",
      "   9.99985337e-01 2.43664942e-07]\n",
      "  [4.06226625e-15 1.00000000e+00 6.47788212e-10 4.31934988e-10\n",
      "   6.11302453e-10 1.28564132e-10 9.18992105e-11 1.53391917e-11\n",
      "   4.72937800e-10 1.61752722e-10]]]\n",
      "['B-ORG', 'O', 'B-MISC', 'O', 'O', 'O', 'B-MISC', 'O']\n"
     ]
    }
   ],
   "source": [
    "ner_model.compile(optimizer=\"adam\", loss=loss)\n",
    "ner_model.fit(train_dataset, epochs=20)\n",
    "print(ner_model.summary())\n",
    "\n",
    "def tokenize_and_convert_to_ids(text):\n",
    "    tokens = text.split()\n",
    "    return lowercase_and_convert_to_ids(tokens)\n",
    "\n",
    "\n",
    "# Sample inference using the trained model\n",
    "sample_input = tokenize_and_convert_to_ids(\n",
    "    \"eu rejects german call to boycott british lamb\"\n",
    ")\n",
    "sample_input = tf.reshape(sample_input, shape=[1, -1])\n",
    "print(sample_input)\n",
    "\n",
    "output = ner_model.predict(sample_input)\n",
    "print(output)\n",
    "prediction = np.argmax(output, axis=-1)[0]\n",
    "prediction = [mapping[i] for i in prediction]\n",
    "\n",
    "# eu -> B-ORG, german -> B-MISC, british -> B-MISC\n",
    "print(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7f25796a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tf.keras.utils.plot_model(ner_model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bc605fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique(listIn):\n",
    " \n",
    "    # initialize a null list\n",
    "    unique_list = []\n",
    " \n",
    "    # traverse for all elements\n",
    "    for x in listIn:\n",
    "        # check if exists in unique_list or not\n",
    "        if x not in unique_list:\n",
    "            unique_list.append(x)\n",
    "    return unique_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1273fcc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 51362 tokens with 5942 phrases; found: 5875 phrases; correct: 4016.\n",
      "accuracy:  66.07%; (non-O)\n",
      "accuracy:  93.17%; precision:  68.36%; recall:  67.59%; FB1:  67.97\n",
      "              LOC: precision:  77.29%; recall:  80.78%; FB1:  79.00  1920\n",
      "             MISC: precision:  74.91%; recall:  65.08%; FB1:  69.65  801\n",
      "              ORG: precision:  57.28%; recall:  64.28%; FB1:  60.58  1505\n",
      "              PER: precision:  64.89%; recall:  58.09%; FB1:  61.30  1649\n",
      "CPU times: user 3.52 s, sys: 1.28 s, total: 4.8 s\n",
      "Wall time: 3.09 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def calculate_metrics(dataset):\n",
    "    all_true_tag_ids, all_predicted_tag_ids = [], []\n",
    "\n",
    "    for x, y in dataset:\n",
    "        output = ner_model.predict(x, verbose=0)\n",
    "        predictions = np.argmax(output, axis=-1)\n",
    "        predictions = np.reshape(predictions, [-1])\n",
    "\n",
    "        true_tag_ids = np.reshape(y, [-1])\n",
    "\n",
    "        mask = (true_tag_ids > 0) & (predictions > 0)\n",
    "        true_tag_ids = true_tag_ids[mask]\n",
    "        predicted_tag_ids = predictions[mask]\n",
    "\n",
    "        all_true_tag_ids.append(true_tag_ids)\n",
    "        all_predicted_tag_ids.append(predicted_tag_ids)\n",
    "\n",
    "    all_true_tag_ids = np.concatenate(all_true_tag_ids)\n",
    "    all_predicted_tag_ids = np.concatenate(all_predicted_tag_ids)\n",
    "\n",
    "    predicted_tags = [mapping[tag] for tag in all_predicted_tag_ids]\n",
    "    real_tags = [mapping[tag] for tag in all_true_tag_ids]\n",
    "\n",
    "    evaluate(real_tags, predicted_tags)\n",
    "\n",
    "\n",
    "calculate_metrics(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd3de850",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}