Text_summarisation_samsum.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "TWo-ooxXojef"
      },
      "outputs": [],
      "source": [
        "!pip install transformers[sentencepiece] datasets sacrebleu rouge_score py7zr -q matplotlib\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s6K8taT_ojeg"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CZAiyMZRojeg",
        "outputId": "2e134310-fbe1-4a29-b80f-478afb1587f9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Thu Sep 28 22:03:11 2023       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 536.99                 Driver Version: 536.99       CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                     TCC/WDDM  | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA GeForce RTX 3050 ...  WDDM  | 00000000:01:00.0 Off |                  N/A |\n",
            "| N/A   55C    P3               8W /  35W |    141MiB /  4096MiB |      6%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|    0   N/A  N/A      7996    C+G   ...2txyewy\\StartMenuExperienceHost.exe    N/A      |\n",
            "|    0   N/A  N/A      8832    C+G   ...nt.CBS_cw5n1h2txyewy\\SearchHost.exe    N/A      |\n",
            "|    0   N/A  N/A     14604    C+G   ...ekyb3d8bbwe\\PhoneExperienceHost.exe    N/A      |\n",
            "|    0   N/A  N/A     16832    C+G   ...Programs\\Microsoft VS Code\\Code.exe    N/A      |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DNmR2POiojeg",
        "outputId": "d2103474-91ba-432d-8419-ff4dc7cceaff"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (2.0.1)\n",
            "Requirement already satisfied: filelock in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch) (4.7.1)\n",
            "Requirement already satisfied: sympy in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch) (3.1)\n",
            "Requirement already satisfied: jinja2 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from sympy->torch) (1.3.0)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\ASUS\\anaconda3\\envs\\Textsummarization\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n",
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     C:\\Users\\ASUS\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "!pip install torch\n",
        "from transformers import pipeline, set_seed\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from datasets import load_dataset\n",
        "import pandas as pd\n",
        "from datasets import load_dataset, load_metric\n",
        "\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "\n",
        "import nltk\n",
        "from nltk.tokenize import sent_tokenize\n",
        "\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "\n",
        "nltk.download(\"punkt\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zu5uNooKojeh",
        "outputId": "46278519-9f03-4178-dac4-d843f85196db"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of PegasusForConditionalGeneration were not initialized from the model checkpoint at google/pegasus-cnn_dailymail and are newly initialized: ['model.decoder.embed_positions.weight', 'model.encoder.embed_positions.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "from datasets import load_metric\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "model_ckpt = \"google/pegasus-cnn_dailymail\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
        "\n",
        "model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(model_ckpt).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1i7ZMI9Aojeh",
        "outputId": "e9d977b6-83e8-4daf-88fe-eea4624d75df"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "DevEZQDHojeh"
      },
      "outputs": [],
      "source": [
        "def generate_batch_sized_chunks(list_of_elements, batch_size):\n",
        "    \"\"\"split the dataset into smaller batches that we can process simultaneously\n",
        "    Yield successive batch-sized chunks from list_of_elements.\"\"\"\n",
        "    for i in range(0, len(list_of_elements), batch_size):\n",
        "        yield list_of_elements[i : i + batch_size]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "49iS-73Nojeh"
      },
      "outputs": [],
      "source": [
        "def calculate_metric_on_test_ds(dataset, metric, model, tokenizer,\n",
        "                               batch_size=16, device=device,\n",
        "                               column_text=\"article\",\n",
        "                               column_summary=\"highlights\"):\n",
        "    article_batches = list(generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
        "    target_batches = list(generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
        "\n",
        "    for article_batch, target_batch in tqdm(\n",
        "        zip(article_batches, target_batches), total=len(article_batches)):\n",
        "\n",
        "        inputs = tokenizer(article_batch, max_length=1024,  truncation=True,\n",
        "                        padding=\"max_length\", return_tensors=\"pt\")\n",
        "\n",
        "        summaries = model.generate(input_ids=inputs[\"input_ids\"].to(device),\n",
        "                         attention_mask=inputs[\"attention_mask\"].to(device),\n",
        "                         length_penalty=0.8, num_beams=8, max_length=128)\n",
        "        ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''\n",
        "\n",
        "        # Finally, we decode the generated texts,\n",
        "        # replace the  token, and add the decoded texts with the references to the metric.\n",
        "        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,\n",
        "                                clean_up_tokenization_spaces=True)\n",
        "               for s in summaries]\n",
        "\n",
        "        decoded_summaries = [d.replace(\"\", \" \") for d in decoded_summaries]\n",
        "\n",
        "\n",
        "        metric.add_batch(predictions=decoded_summaries, references=target_batch)\n",
        "\n",
        "    #  Finally compute and return the ROUGE scores.\n",
        "    score = metric.compute()\n",
        "    return score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LpOZZIgvojei",
        "outputId": "70752f4e-9ced-4dc2-cd4e-8663280af254"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Found cached dataset samsum (C:/Users/ASUS/.cache/huggingface/datasets/samsum/samsum/0.0.0/f1d7c6b7353e6de335d444e424dc002ef70d1277109031327bc9cc6af5d3d46e)\n",
            "100%|██████████| 3/3 [00:00<00:00, 91.20it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Split lengths: [14732, 819, 818]\n",
            "Features: ['id', 'dialogue', 'summary']\n",
            "\n",
            "Dialogue:\n",
            "Eric: MACHINE!\n",
            "Rob: That's so gr8!\n",
            "Eric: I know! And shows how Americans see Russian ;)\n",
            "Rob: And it's really funny!\n",
            "Eric: I know! I especially like the train part!\n",
            "Rob: Hahaha! No one talks to the machine like that!\n",
            "Eric: Is this his only stand-up?\n",
            "Rob: Idk. I'll check.\n",
            "Eric: Sure.\n",
            "Rob: Turns out no! There are some of his stand-ups on youtube.\n",
            "Eric: Gr8! I'll watch them now!\n",
            "Rob: Me too!\n",
            "Eric: MACHINE!\n",
            "Rob: MACHINE!\n",
            "Eric: TTYL?\n",
            "Rob: Sure :)\n",
            "\n",
            "Summary:\n",
            "Eric and Rob are going to watch a stand-up on youtube.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "dataset_samsum = load_dataset(\"samsum\")\n",
        "\n",
        "split_lengths = [len(dataset_samsum[split])for split in dataset_samsum]\n",
        "\n",
        "print(f\"Split lengths: {split_lengths}\")\n",
        "print(f\"Features: {dataset_samsum['train'].column_names}\")\n",
        "print(\"\\nDialogue:\")\n",
        "\n",
        "print(dataset_samsum[\"test\"][1][\"dialogue\"])\n",
        "\n",
        "print(\"\\nSummary:\")\n",
        "\n",
        "print(dataset_samsum[\"test\"][1][\"summary\"])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "wGl804zQojei",
        "outputId": "67feb8a9-5fba-4f52-a2ae-b65cae109066"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"Hannah: Hey, do you have Betty's number?\\nAmanda: Lemme check\\nHannah: <file_gif>\\nAmanda: Sorry, can't find it.\\nAmanda: Ask Larry\\nAmanda: He called her last time we were at the park together\\nHannah: I don't know him well\\nHannah: <file_gif>\\nAmanda: Don't be shy, he's very nice\\nHannah: If you say so..\\nHannah: I'd rather you texted him\\nAmanda: Just text him 🙂\\nHannah: Urgh.. Alright\\nHannah: Bye\\nAmanda: Bye bye\""
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset_samsum['test'][0]['dialogue']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3QeucXeoojei",
        "outputId": "89de8417-cae5-471f-ba1a-8486743347d8"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of PegasusForConditionalGeneration were not initialized from the model checkpoint at google/pegasus-cnn_dailymail and are newly initialized: ['model.decoder.embed_positions.weight', 'model.encoder.embed_positions.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "Your max_length is set to 128, but your input_length is only 122. Since this is a summarization task, where outputs shorter than the input are typically wanted, you might consider decreasing max_length manually, e.g. summarizer('...', max_length=61)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[{'summary_text': \"Amanda: Ask Larry Amanda: He called her last time we were at the park together .<n>Hannah: I'd rather you texted him .<n>Amanda: Just text him .\"}]\n"
          ]
        }
      ],
      "source": [
        "pipe = pipeline('summarization', model = model_ckpt )\n",
        "\n",
        "pipe_out = pipe(dataset_samsum['test'][0]['dialogue'] )\n",
        "\n",
        "print(pipe_out)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JOxhVBvkojei",
        "outputId": "94177425-f19f-469e-fa26-5e76446b7ffe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Amanda: Ask Larry Amanda: He called her last time we were at the park together.\n",
            "<n>Hannah: I'd rather you texted him.\n",
            "<n>Amanda: Just text him.\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(pipe_out[0]['summary_text'].replace(\" .\", \".\\n\"))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pI37fLmeojei"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "kF8BSiWnojei"
      },
      "outputs": [],
      "source": [
        "#rouge_metric = load_metric('rouge')\n",
        "#score = calculate_metric_on_test_ds(dataset_samsum['test'], rouge_metric, model_pegasus, tokenizer, column_text = 'dialogue', column_summary='summary', batch_size=8)\n",
        "#rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names )\n",
        "#pd.DataFrame(rouge_dict, index = ['pegasus'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xv0e2Tcbojej"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "e64e13ecc92348799013bb821099cfe3",
            "363180cacc0847109b210288b42e0a13",
            "c3924919e7cb4ddba97fa7822bb7037c",
            "de71d56ccd77421ca8a8008218c39b5f",
            "de14f90e96304eba96dd7f0534461051",
            "b1420907af874518a326988189ca8bf4",
            "eabb395591d3450dbc9fcc92a7438adc",
            "7d7ab051cb594a25b290ac7f1f5ffa64",
            "8a46dca1d5514c27bdfd0a9a07ce2aed",
            "42264bae489542e48a3b775f8cf8933b",
            "82e6b25a863445dfa96839553c85cc2f"
          ]
        },
        "id": "-wLo5xfK4TOq",
        "outputId": "a2491698-48e4-44f7-fdae-c2bf274e64ee"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Map:   0%|          | 0/14732 [00:00<?, ? examples/s]c:\\Users\\ASUS\\anaconda3\\envs\\Textsummarization\\lib\\site-packages\\transformers\\tokenization_utils_base.py:3660: UserWarning: `as_target_tokenizer` is deprecated and will be removed in v5 of Transformers. You can tokenize your labels by using the argument `text_target` of the regular `__call__` method (either in the same call as your input texts if you use the same keyword arguments, or in a separate call.\n",
            "  warnings.warn(\n",
            "Loading cached processed dataset at C:\\Users\\ASUS\\.cache\\huggingface\\datasets\\samsum\\samsum\\0.0.0\\f1d7c6b7353e6de335d444e424dc002ef70d1277109031327bc9cc6af5d3d46e\\cache-92087251849db6b9.arrow\n",
            "Loading cached processed dataset at C:\\Users\\ASUS\\.cache\\huggingface\\datasets\\samsum\\samsum\\0.0.0\\f1d7c6b7353e6de335d444e424dc002ef70d1277109031327bc9cc6af5d3d46e\\cache-41e05d9b612d6dc8.arrow\n"
          ]
        }
      ],
      "source": [
        "\n",
        "def convert_examples_to_features(example_batch):\n",
        "    input_encodings = tokenizer(example_batch['dialogue'] , max_length = 1024, truncation = True )\n",
        "\n",
        "    with tokenizer.as_target_tokenizer():\n",
        "        target_encodings = tokenizer(example_batch['summary'], max_length = 128, truncation = True )\n",
        "\n",
        "    return {\n",
        "        'input_ids' : input_encodings['input_ids'],\n",
        "        'attention_mask': input_encodings['attention_mask'],\n",
        "        'labels': target_encodings['input_ids']\n",
        "    }\n",
        "\n",
        "dataset_samsum_pt = dataset_samsum.map(convert_examples_to_features, batched = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5RX5-JJ0ojej",
        "outputId": "e055f91c-adc1-4fef-8f52-dd863021c69b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: accelerate in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (0.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (23.1)\n",
            "Requirement already satisfied: psutil in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (5.9.0)\n",
            "Requirement already satisfied: pyyaml in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (6.0)\n",
            "Requirement already satisfied: torch>=1.10.0 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (2.0.1)\n",
            "Requirement already satisfied: huggingface-hub in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from accelerate) (0.15.1)\n",
            "Requirement already satisfied: filelock in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch>=1.10.0->accelerate) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch>=1.10.0->accelerate) (4.7.1)\n",
            "Requirement already satisfied: sympy in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch>=1.10.0->accelerate) (3.1)\n",
            "Requirement already satisfied: jinja2 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: fsspec in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from huggingface-hub->accelerate) (2023.4.0)\n",
            "Requirement already satisfied: requests in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from huggingface-hub->accelerate) (4.65.0)\n",
            "Requirement already satisfied: colorama in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from tqdm>=4.42.1->huggingface-hub->accelerate) (0.4.6)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from requests->huggingface-hub->accelerate) (2.0.4)\n",
            "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from requests->huggingface-hub->accelerate) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from requests->huggingface-hub->accelerate) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from requests->huggingface-hub->accelerate) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in c:\\users\\asus\\anaconda3\\envs\\textsummarization\\lib\\site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install accelerate -U\n",
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RAaBqrlb8FHS",
        "outputId": "65e546aa-d620-4a75-e06d-b748a4b1bd7b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'id': '13818513',\n",
              " 'dialogue': \"Amanda: I baked  cookies. Do you want some?\\r\\nJerry: Sure!\\r\\nAmanda: I'll bring you tomorrow :-)\",\n",
              " 'summary': 'Amanda baked cookies and will bring Jerry some tomorrow.',\n",
              " 'input_ids': [12195,\n",
              "  151,\n",
              "  125,\n",
              "  7091,\n",
              "  3659,\n",
              "  107,\n",
              "  842,\n",
              "  119,\n",
              "  245,\n",
              "  181,\n",
              "  152,\n",
              "  10508,\n",
              "  151,\n",
              "  7435,\n",
              "  147,\n",
              "  12195,\n",
              "  151,\n",
              "  125,\n",
              "  131,\n",
              "  267,\n",
              "  650,\n",
              "  119,\n",
              "  3469,\n",
              "  29344,\n",
              "  1],\n",
              " 'attention_mask': [1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1,\n",
              "  1],\n",
              " 'labels': [12195, 7091, 3659, 111, 138, 650, 10508, 181, 3469, 107, 1]}"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset_samsum_pt['train'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "qVrg6fUrojej"
      },
      "outputs": [],
      "source": [
        "from transformers import TrainingArguments, Trainer\n",
        "\n",
        "trainer_args = TrainingArguments(\n",
        "    output_dir='pegasus-samsum', num_train_epochs=1, warmup_steps=500,\n",
        "    per_device_train_batch_size=1, per_device_eval_batch_size=1,\n",
        "    weight_decay=0.01, logging_steps=10,\n",
        "    evaluation_strategy='steps', eval_steps=500, save_steps=1e6,\n",
        "    gradient_accumulation_steps=16\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "r2t_ucUJojej"
      },
      "outputs": [],
      "source": [
        "\n",
        "trainer = Trainer(model=model_pegasus, args=trainer_args,\n",
        "                  tokenizer=tokenizer, data_collator=seq2seq_data_collator,\n",
        "                  train_dataset=dataset_samsum_pt[\"train\"],\n",
        "                  eval_dataset=dataset_samsum_pt[\"validation\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vJTO6bbyFcG0",
        "outputId": "6a59cd2b-bad7-4d5f-a70d-11510564f701"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 0/920 [00:00<?, ?it/s]You're using a PegasusTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "  1%|          | 10/920 [07:29<11:42:45, 46.34s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 3.1382, 'learning_rate': 1.0000000000000002e-06, 'epoch': 0.01}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  2%|▏         | 20/920 [15:27<11:55:51, 47.72s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 3.0794, 'learning_rate': 2.0000000000000003e-06, 'epoch': 0.02}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3%|▎         | 30/920 [22:25<9:51:06, 39.85s/it] "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.8823, 'learning_rate': 3e-06, 'epoch': 0.03}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  4%|▍         | 40/920 [30:00<11:08:57, 45.61s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.8974, 'learning_rate': 4.000000000000001e-06, 'epoch': 0.04}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  5%|▌         | 50/920 [37:05<10:00:45, 41.43s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.8281, 'learning_rate': 5e-06, 'epoch': 0.05}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  7%|▋         | 60/920 [44:17<10:16:08, 42.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.7812, 'learning_rate': 6e-06, 'epoch': 0.07}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  8%|▊         | 70/920 [51:42<10:33:53, 44.74s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.5177, 'learning_rate': 7.000000000000001e-06, 'epoch': 0.08}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  9%|▊         | 80/920 [58:18<8:59:01, 38.50s/it] "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.6783, 'learning_rate': 8.000000000000001e-06, 'epoch': 0.09}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 10%|▉         | 90/920 [1:05:39<9:59:23, 43.33s/it] "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.4155, 'learning_rate': 9e-06, 'epoch': 0.1}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 11%|█         | 100/920 [1:13:08<10:20:44, 45.42s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.4785, 'learning_rate': 1e-05, 'epoch': 0.11}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 12%|█▏        | 110/920 [1:20:28<10:11:32, 45.30s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.3184, 'learning_rate': 1.1000000000000001e-05, 'epoch': 0.12}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 120/920 [1:27:42<9:37:58, 43.35s/it] "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.1031, 'learning_rate': 1.2e-05, 'epoch': 0.13}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 14%|█▍        | 130/920 [1:34:55<9:25:02, 42.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.253, 'learning_rate': 1.3000000000000001e-05, 'epoch': 0.14}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 15%|█▌        | 140/920 [1:42:42<10:00:28, 46.19s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.0861, 'learning_rate': 1.4000000000000001e-05, 'epoch': 0.15}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 16%|█▋        | 150/920 [1:50:05<9:32:05, 44.58s/it] "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.0042, 'learning_rate': 1.5e-05, 'epoch': 0.16}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 17%|█▋        | 160/920 [1:57:29<9:30:34, 45.05s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.0297, 'learning_rate': 1.6000000000000003e-05, 'epoch': 0.17}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 18%|█▊        | 170/920 [2:04:48<8:49:49, 42.39s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.0642, 'learning_rate': 1.7000000000000003e-05, 'epoch': 0.18}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 20%|█▉        | 180/920 [2:12:22<9:17:11, 45.18s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 2.0554, 'learning_rate': 1.8e-05, 'epoch': 0.2}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 21%|██        | 190/920 [2:19:28<8:42:30, 42.95s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.9065, 'learning_rate': 1.9e-05, 'epoch': 0.21}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 22%|██▏       | 200/920 [2:27:13<8:38:13, 43.19s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.9788, 'learning_rate': 2e-05, 'epoch': 0.22}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 23%|██▎       | 210/920 [2:34:28<8:05:04, 40.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.8685, 'learning_rate': 2.1e-05, 'epoch': 0.23}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 24%|██▍       | 220/920 [2:41:53<8:49:57, 45.42s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.8106, 'learning_rate': 2.2000000000000003e-05, 'epoch': 0.24}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 25%|██▌       | 230/920 [2:49:10<8:13:08, 42.88s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.8761, 'learning_rate': 2.3000000000000003e-05, 'epoch': 0.25}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 26%|██▌       | 240/920 [2:56:46<8:37:40, 45.68s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.8151, 'learning_rate': 2.4e-05, 'epoch': 0.26}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 250/920 [3:03:57<8:09:29, 43.83s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.8545, 'learning_rate': 2.5e-05, 'epoch': 0.27}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 28%|██▊       | 260/920 [3:11:38<8:04:06, 44.01s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.9141, 'learning_rate': 2.6000000000000002e-05, 'epoch': 0.28}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 29%|██▉       | 270/920 [3:18:48<7:30:23, 41.57s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7736, 'learning_rate': 2.7000000000000002e-05, 'epoch': 0.29}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 30%|███       | 280/920 [3:26:12<7:54:20, 44.47s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7433, 'learning_rate': 2.8000000000000003e-05, 'epoch': 0.3}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 290/920 [3:33:18<7:51:58, 44.95s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.683, 'learning_rate': 2.9e-05, 'epoch': 0.31}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 300/920 [3:40:26<7:07:08, 41.34s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7371, 'learning_rate': 3e-05, 'epoch': 0.33}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 34%|███▎      | 310/920 [3:47:24<6:55:10, 40.84s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7525, 'learning_rate': 3.1e-05, 'epoch': 0.34}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 35%|███▍      | 320/920 [3:54:41<7:22:05, 44.21s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7611, 'learning_rate': 3.2000000000000005e-05, 'epoch': 0.35}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 36%|███▌      | 330/920 [4:01:40<6:52:14, 41.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.78, 'learning_rate': 3.3e-05, 'epoch': 0.36}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 340/920 [4:08:47<7:00:25, 43.49s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6784, 'learning_rate': 3.4000000000000007e-05, 'epoch': 0.37}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 38%|███▊      | 350/920 [4:16:21<7:08:59, 45.16s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6641, 'learning_rate': 3.5e-05, 'epoch': 0.38}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 39%|███▉      | 360/920 [4:23:28<6:41:29, 43.02s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.678, 'learning_rate': 3.6e-05, 'epoch': 0.39}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 40%|████      | 370/920 [4:30:35<6:48:59, 44.62s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6782, 'learning_rate': 3.7e-05, 'epoch': 0.4}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 41%|████▏     | 380/920 [4:38:03<6:32:04, 43.56s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7276, 'learning_rate': 3.8e-05, 'epoch': 0.41}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 390/920 [4:45:39<6:17:24, 42.72s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.789, 'learning_rate': 3.9000000000000006e-05, 'epoch': 0.42}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 43%|████▎     | 400/920 [4:52:44<6:12:32, 42.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6157, 'learning_rate': 4e-05, 'epoch': 0.43}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 45%|████▍     | 410/920 [4:59:47<6:02:41, 42.67s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.581, 'learning_rate': 4.1e-05, 'epoch': 0.45}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 46%|████▌     | 420/920 [5:07:21<6:20:27, 45.66s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6933, 'learning_rate': 4.2e-05, 'epoch': 0.46}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 430/920 [5:14:34<5:56:13, 43.62s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6605, 'learning_rate': 4.3e-05, 'epoch': 0.47}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 48%|████▊     | 440/920 [5:21:41<5:46:22, 43.30s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.693, 'learning_rate': 4.4000000000000006e-05, 'epoch': 0.48}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 49%|████▉     | 450/920 [5:29:07<5:58:50, 45.81s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6541, 'learning_rate': 4.5e-05, 'epoch': 0.49}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 50%|█████     | 460/920 [5:36:19<5:37:04, 43.97s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6139, 'learning_rate': 4.600000000000001e-05, 'epoch': 0.5}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 51%|█████     | 470/920 [5:43:23<5:23:53, 43.18s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6253, 'learning_rate': 4.7e-05, 'epoch': 0.51}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 52%|█████▏    | 480/920 [5:50:43<5:06:06, 41.74s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6636, 'learning_rate': 4.8e-05, 'epoch': 0.52}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 490/920 [5:57:38<4:53:11, 40.91s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6103, 'learning_rate': 4.9e-05, 'epoch': 0.53}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 54%|█████▍    | 500/920 [6:05:04<5:02:45, 43.25s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7102, 'learning_rate': 5e-05, 'epoch': 0.54}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "                                                     \n",
            " 54%|█████▍    | 500/920 [6:14:15<5:02:45, 43.25s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'eval_loss': 1.4890581369400024, 'eval_runtime': 550.7934, 'eval_samples_per_second': 1.485, 'eval_steps_per_second': 1.485, 'epoch': 0.54}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 55%|█████▌    | 510/920 [6:21:25<5:43:30, 50.27s/it]  "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6135, 'learning_rate': 4.880952380952381e-05, 'epoch': 0.55}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 57%|█████▋    | 520/920 [6:28:11<4:37:10, 41.58s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7072, 'learning_rate': 4.761904761904762e-05, 'epoch': 0.56}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 530/920 [6:35:38<4:59:27, 46.07s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6695, 'learning_rate': 4.642857142857143e-05, 'epoch': 0.58}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 59%|█████▊    | 540/920 [6:43:25<4:35:34, 43.51s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6137, 'learning_rate': 4.523809523809524e-05, 'epoch': 0.59}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|█████▉    | 550/920 [6:50:33<4:17:59, 41.84s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.648, 'learning_rate': 4.404761904761905e-05, 'epoch': 0.6}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 61%|██████    | 560/920 [6:57:41<4:12:00, 42.00s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6179, 'learning_rate': 4.2857142857142856e-05, 'epoch': 0.61}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 62%|██████▏   | 570/920 [7:05:09<4:05:03, 42.01s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.578, 'learning_rate': 4.166666666666667e-05, 'epoch': 0.62}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 580/920 [7:12:58<4:16:44, 45.31s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7023, 'learning_rate': 4.047619047619048e-05, 'epoch': 0.63}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 64%|██████▍   | 590/920 [7:19:29<3:39:11, 39.85s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5825, 'learning_rate': 3.928571428571429e-05, 'epoch': 0.64}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 65%|██████▌   | 600/920 [7:26:39<3:50:17, 43.18s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6765, 'learning_rate': 3.809523809523809e-05, 'epoch': 0.65}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 66%|██████▋   | 610/920 [7:33:46<3:50:04, 44.53s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6221, 'learning_rate': 3.690476190476191e-05, 'epoch': 0.66}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 67%|██████▋   | 620/920 [7:41:58<4:00:08, 48.03s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6205, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.67}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 630/920 [7:49:16<3:32:23, 43.94s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.7048, 'learning_rate': 3.4523809523809526e-05, 'epoch': 0.68}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 70%|██████▉   | 640/920 [7:56:41<3:28:55, 44.77s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5792, 'learning_rate': 3.3333333333333335e-05, 'epoch': 0.7}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 71%|███████   | 650/920 [8:03:23<2:59:38, 39.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5312, 'learning_rate': 3.2142857142857144e-05, 'epoch': 0.71}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 72%|███████▏  | 660/920 [8:10:41<3:09:46, 43.79s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6444, 'learning_rate': 3.095238095238095e-05, 'epoch': 0.72}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 73%|███████▎  | 670/920 [8:17:51<2:52:15, 41.34s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6985, 'learning_rate': 2.9761904761904762e-05, 'epoch': 0.73}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 74%|███████▍  | 680/920 [8:24:42<2:49:54, 42.48s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5376, 'learning_rate': 2.857142857142857e-05, 'epoch': 0.74}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 75%|███████▌  | 690/920 [8:32:15<3:01:39, 47.39s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6179, 'learning_rate': 2.7380952380952383e-05, 'epoch': 0.75}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 76%|███████▌  | 700/920 [8:39:42<2:42:11, 44.23s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6138, 'learning_rate': 2.6190476190476192e-05, 'epoch': 0.76}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 77%|███████▋  | 710/920 [8:46:58<2:23:17, 40.94s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6478, 'learning_rate': 2.5e-05, 'epoch': 0.77}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 78%|███████▊  | 720/920 [8:54:28<2:27:09, 44.15s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6452, 'learning_rate': 2.380952380952381e-05, 'epoch': 0.78}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 730/920 [9:01:33<2:07:35, 40.29s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5242, 'learning_rate': 2.261904761904762e-05, 'epoch': 0.79}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 80%|████████  | 740/920 [9:09:25<2:22:23, 47.46s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.589, 'learning_rate': 2.1428571428571428e-05, 'epoch': 0.8}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 82%|████████▏ | 750/920 [9:16:32<2:05:25, 44.27s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5657, 'learning_rate': 2.023809523809524e-05, 'epoch': 0.81}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 83%|████████▎ | 760/920 [9:23:32<1:58:19, 44.37s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6248, 'learning_rate': 1.9047619047619046e-05, 'epoch': 0.83}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 84%|████████▎ | 770/920 [9:30:38<1:49:14, 43.69s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5945, 'learning_rate': 1.785714285714286e-05, 'epoch': 0.84}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 85%|████████▍ | 780/920 [9:38:20<1:48:01, 46.30s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6642, 'learning_rate': 1.6666666666666667e-05, 'epoch': 0.85}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 86%|████████▌ | 790/920 [9:45:23<1:31:43, 42.33s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5738, 'learning_rate': 1.5476190476190476e-05, 'epoch': 0.86}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 87%|████████▋ | 800/920 [9:52:28<1:23:25, 41.71s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.553, 'learning_rate': 1.4285714285714285e-05, 'epoch': 0.87}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 88%|████████▊ | 810/920 [10:00:25<1:26:45, 47.33s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6526, 'learning_rate': 1.3095238095238096e-05, 'epoch': 0.88}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 820/920 [10:08:13<1:20:43, 48.43s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6425, 'learning_rate': 1.1904761904761905e-05, 'epoch': 0.89}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 90%|█████████ | 830/920 [10:15:03<1:02:22, 41.58s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5358, 'learning_rate': 1.0714285714285714e-05, 'epoch': 0.9}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 91%|█████████▏| 840/920 [10:22:46<1:00:18, 45.23s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5749, 'learning_rate': 9.523809523809523e-06, 'epoch': 0.91}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 92%|█████████▏| 850/920 [10:29:56<49:57, 42.83s/it]  "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5691, 'learning_rate': 8.333333333333334e-06, 'epoch': 0.92}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 93%|█████████▎| 860/920 [10:37:37<45:55, 45.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6939, 'learning_rate': 7.142857142857143e-06, 'epoch': 0.93}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 870/920 [10:44:31<34:27, 41.34s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5322, 'learning_rate': 5.9523809523809525e-06, 'epoch': 0.94}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 96%|█████████▌| 880/920 [10:51:46<28:45, 43.14s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5842, 'learning_rate': 4.7619047619047615e-06, 'epoch': 0.96}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 97%|█████████▋| 890/920 [10:58:48<21:13, 42.44s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5306, 'learning_rate': 3.5714285714285714e-06, 'epoch': 0.97}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 98%|█████████▊| 900/920 [11:06:11<15:19, 45.98s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6285, 'learning_rate': 2.3809523809523808e-06, 'epoch': 0.98}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 99%|█████████▉| 910/920 [11:13:25<07:05, 42.56s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.6005, 'learning_rate': 1.1904761904761904e-06, 'epoch': 0.99}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 920/920 [11:20:52<00:00, 44.40s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'loss': 1.5937, 'learning_rate': 0.0, 'epoch': 1.0}\n",
            "{'train_runtime': 40852.0207, 'train_samples_per_second': 0.361, 'train_steps_per_second': 0.023, 'train_loss': 1.8250145041424295, 'epoch': 1.0}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=920, training_loss=1.8250145041424295, metrics={'train_runtime': 40852.0207, 'train_samples_per_second': 0.361, 'train_steps_per_second': 0.023, 'train_loss': 1.8250145041424295, 'epoch': 1.0})"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "trainer.train()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "h3q8noYFFcPz"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 410/410 [3:35:06<00:00, 31.48s/it]  \n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>rouge1</th>\n",
              "      <th>rouge2</th>\n",
              "      <th>rougeL</th>\n",
              "      <th>rougeLsum</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>pegasus</th>\n",
              "      <td>0.018524</td>\n",
              "      <td>0.000332</td>\n",
              "      <td>0.018384</td>\n",
              "      <td>0.018436</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           rouge1    rouge2    rougeL  rougeLsum\n",
              "pegasus  0.018524  0.000332  0.018384   0.018436"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "rouge_metric = load_metric('rouge')\n",
        "score = calculate_metric_on_test_ds(\n",
        "    dataset_samsum['test'], rouge_metric, trainer.model, tokenizer, batch_size = 2, column_text = 'dialogue', column_summary= 'summary'\n",
        ")\n",
        "\n",
        "rouge_names = [\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"]\n",
        "rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names )\n",
        "\n",
        "pd.DataFrame(rouge_dict, index = [f'pegasus'] )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "GoZObU_hFcUd"
      },
      "outputs": [],
      "source": [
        "\n",
        "## Save model\n",
        "model_pegasus.save_pretrained(\"pegasus-samsum-model\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "dPYeyLtVElyk"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('tokenizer\\\\tokenizer_config.json',\n",
              " 'tokenizer\\\\special_tokens_map.json',\n",
              " 'tokenizer\\\\spiece.model',\n",
              " 'tokenizer\\\\added_tokens.json',\n",
              " 'tokenizer\\\\tokenizer.json')"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "## Save tokenizer\n",
        "tokenizer.save_pretrained(\"tokenizer\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "VSh4gUaAEl9Y"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:datasets.builder:Found cached dataset samsum (C:/Users/ASUS/.cache/huggingface/datasets/samsum/samsum/0.0.0/f1d7c6b7353e6de335d444e424dc002ef70d1277109031327bc9cc6af5d3d46e)\n",
            "100%|██████████| 3/3 [00:00<00:00, 124.68it/s]\n"
          ]
        }
      ],
      "source": [
        "dataset_samsum = load_dataset(\"samsum\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "NC8H_IeD_2nE"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"tokenizer\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "UDg_1m4Gojej"
      },
      "outputs": [],
      "source": [
        "\n",
        "sample_text = dataset_samsum[\"test\"][0][\"dialogue\"]\n",
        "\n",
        "reference = dataset_samsum[\"test\"][0][\"summary\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "7vWcpK36ojej"
      },
      "outputs": [],
      "source": [
        "\n",
        "gen_kwargs = {\"length_penalty\": 0.8, \"num_beams\":8, \"max_length\": 128}\n",
        "\n",
        "pipe = pipeline(\"summarization\", model=\"pegasus-samsum-model\",tokenizer=tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "WgsQd7Cwojej"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Your max_length is set to 128, but your input_length is only 122. Since this is a summarization task, where outputs shorter than the input are typically wanted, you might consider decreasing max_length manually, e.g. summarizer('...', max_length=61)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dialogue:\n",
            "Hannah: Hey, do you have Betty's number?\n",
            "Amanda: Lemme check\n",
            "Hannah: <file_gif>\n",
            "Amanda: Sorry, can't find it.\n",
            "Amanda: Ask Larry\n",
            "Amanda: He called her last time we were at the park together\n",
            "Hannah: I don't know him well\n",
            "Hannah: <file_gif>\n",
            "Amanda: Don't be shy, he's very nice\n",
            "Hannah: If you say so..\n",
            "Hannah: I'd rather you texted him\n",
            "Amanda: Just text him 🙂\n",
            "Hannah: Urgh.. Alright\n",
            "Hannah: Bye\n",
            "Amanda: Bye bye\n",
            "\n",
            "Reference Summary:\n",
            "Hannah needs Betty's number but Amanda doesn't have it. She needs to contact Larry.\n",
            "\n",
            "Model Summary:\n",
            "Amanda can't find Betty's number. Larry called Betty last time they were at the park together. Hannah wants Amanda to text Larry. Amanda will text Larry.\n"
          ]
        }
      ],
      "source": [
        "print(\"Dialogue:\")\n",
        "print(sample_text)\n",
        "\n",
        "\n",
        "print(\"\\nReference Summary:\")\n",
        "print(reference)\n",
        "\n",
        "\n",
        "print(\"\\nModel Summary:\")\n",
        "print(pipe(sample_text, **gen_kwargs)[0][\"summary_text\"])\n",
        "     "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "ombgWXfWB8Ei"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "c:\\Users\\ASUS\\Downloads\n"
          ]
        }
      ],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yihht7SbB8NW"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lpfh4NT8B8P4"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-kfQ4fD3B8SJ"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5pTN1fBKCosS"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5_V2NRPNCo0G"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-e2qWy86Co2e"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1qq5OOGqCo4v"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4mlQ1gBACo6y"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    },
    "orig_nbformat": 4,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "363180cacc0847109b210288b42e0a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b1420907af874518a326988189ca8bf4",
            "placeholder": "​",
            "style": "IPY_MODEL_eabb395591d3450dbc9fcc92a7438adc",
            "value": "Map: 100%"
          }
        },
        "42264bae489542e48a3b775f8cf8933b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d7ab051cb594a25b290ac7f1f5ffa64": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82e6b25a863445dfa96839553c85cc2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8a46dca1d5514c27bdfd0a9a07ce2aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b1420907af874518a326988189ca8bf4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3924919e7cb4ddba97fa7822bb7037c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d7ab051cb594a25b290ac7f1f5ffa64",
            "max": 818,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8a46dca1d5514c27bdfd0a9a07ce2aed",
            "value": 818
          }
        },
        "de14f90e96304eba96dd7f0534461051": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de71d56ccd77421ca8a8008218c39b5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42264bae489542e48a3b775f8cf8933b",
            "placeholder": "​",
            "style": "IPY_MODEL_82e6b25a863445dfa96839553c85cc2f",
            "value": " 818/818 [00:00&lt;00:00, 1036.66 examples/s]"
          }
        },
        "e64e13ecc92348799013bb821099cfe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_363180cacc0847109b210288b42e0a13",
              "IPY_MODEL_c3924919e7cb4ddba97fa7822bb7037c",
              "IPY_MODEL_de71d56ccd77421ca8a8008218c39b5f"
            ],
            "layout": "IPY_MODEL_de14f90e96304eba96dd7f0534461051"
          }
        },
        "eabb395591d3450dbc9fcc92a7438adc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}