selecting_incorrect_option.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_oNWvd4h5tyy",
        "outputId": "ef751430-c6e9-4773-9106-910e46709a61"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting pyarrow-hotfix (from datasets)\n",
            "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.19.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.3)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.18.0->datasets) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.18.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: pyarrow-hotfix, dill, multiprocess, datasets\n",
            "Successfully installed datasets-2.15.0 dill-0.3.7 multiprocess-0.70.15 pyarrow-hotfix-0.6\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FeGOxVSo5-Ad",
        "outputId": "eb0bb308-605f-4e06-ebb8-ed055c424711"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting together\n",
            "  Downloading together-0.2.8-py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.6/42.6 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydantic<3.0.0,>=2.5.0 (from together)\n",
            "  Downloading pydantic-2.5.2-py3-none-any.whl (381 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m381.9/381.9 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3.0.0,>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from together) (2.31.0)\n",
            "Collecting sseclient-py<2.0.0,>=1.7.2 (from together)\n",
            "  Downloading sseclient_py-1.8.0-py2.py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from together) (0.9.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from together) (4.66.1)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from together) (0.9.0)\n",
            "Collecting annotated-types>=0.4.0 (from pydantic<3.0.0,>=2.5.0->together)\n",
            "  Downloading annotated_types-0.6.0-py3-none-any.whl (12 kB)\n",
            "Collecting pydantic-core==2.14.5 (from pydantic<3.0.0,>=2.5.0->together)\n",
            "  Downloading pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m28.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.6.1 (from pydantic<3.0.0,>=2.5.0->together)\n",
            "  Downloading typing_extensions-4.8.0-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->together) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->together) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->together) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->together) (2023.11.17)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.9.0->together) (8.1.7)\n",
            "Installing collected packages: sseclient-py, typing-extensions, annotated-types, pydantic-core, pydantic, together\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.5.0\n",
            "    Uninstalling typing_extensions-4.5.0:\n",
            "      Successfully uninstalled typing_extensions-4.5.0\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 1.10.13\n",
            "    Uninstalling pydantic-1.10.13:\n",
            "      Successfully uninstalled pydantic-1.10.13\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires fastapi, which is not installed.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "lida 0.0.10 requires uvicorn, which is not installed.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires openai, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed annotated-types-0.6.0 pydantic-2.5.2 pydantic-core-2.14.5 sseclient-py-1.8.0 together-0.2.8 typing-extensions-4.8.0\n"
          ]
        }
      ],
      "source": [
        "!pip install together"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iFR5wa-55xOE"
      },
      "outputs": [],
      "source": [
        "import datasets\n",
        "import together\n",
        "import json\n",
        "import tqdm\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2hFYwK75fHG"
      },
      "source": [
        "### Exlpore if the model api is able to generate different answers for different runs/temperature settings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eTssdrRL57Sl"
      },
      "outputs": [],
      "source": [
        "together.api_key = \"\"\n",
        "\n",
        "\n",
        "def togther_api(prompt, option = None, verbose = False, model = \"togethercomputer/llama-2-13b-chat\"):\n",
        "    if option == None:\n",
        "        prompt = \"<human>:\" + prompt\n",
        "    else:\n",
        "        prompt = \"<human>:\" + prompt + \"\\n<bot>: The correct answer is \" + option + \"\\n<human> Q: Why is that?\"\n",
        "\n",
        "    if verbose:\n",
        "        print (prompt)\n",
        "\n",
        "    output = together.Complete.create(\n",
        "        prompt=prompt,\n",
        "        model=model,\n",
        "        max_tokens=1024,\n",
        "        temperature=0.7,\n",
        "        top_k=50,\n",
        "        top_p=0.7,\n",
        "        repetition_penalty=1,\n",
        "        stop=['<human>', '\\n\\n']\n",
        "    )\n",
        "    return (output['output']['choices'][0]['text'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "3331fb4366404d58a16e7d20c0bd9097",
            "43a41c9e370a4c91b449a07b18e6f1a1",
            "5db1eecb53e04f59ab90284612472c78",
            "3277b6349a0040d986d6b7a39e9139b6",
            "f6723587d1ae4ccd96d12a85294e9e0c",
            "f04b13c4dfe345a59f0a17f509c82bcf",
            "62ea33cfc2c94bc284528b08d4b8f3c5",
            "2ed59712d55146d599c0e71afe142edc",
            "641fda76a3a445ea89612cb18122c933",
            "a644ddc949fd43b4a6f06a831a417a9c",
            "5243c34d06c94c6cbfc7a969dd1ef052",
            "cd41ff734ca148f59afaa74fd11fc3c6",
            "59467a6cbb024d0da5673f24d56672f4",
            "cb66852d6fe145fb847d2ae8c451e924",
            "087b4b5119e94f58b41d80fad5d806e3",
            "b507a0731fce4a4284cdb119bfcabb44",
            "45f6e8fac26c41b7b2decf0efe18890a",
            "be47fc28fa454b948bc73e78f7359bc5",
            "2f15c408406f4dc2acc12f6b0b550bb1",
            "13cc6346fc3a4b81bf5f3e4b5ca826dd",
            "64a103b4c5d043779a5312f48cdbf591",
            "90a5a88b07a041ecbde0dbb3813c4b8e",
            "43ef7e4cacbb420a9ef05feacd13a27e",
            "2e806f25f6354388b5bc462c369cb8ae",
            "080b9a400c884b01b3a58c7c81520413",
            "07e67c2ea89c4b73b58ffed70b1e49ed",
            "febc5b7b0a6a49259c9eb0562def85d9",
            "c9c7c55cf92e4e0ca682799652e5a9b9",
            "5a1fbffb723f4e89a7638d280dc21f9a",
            "69433c9cdb204888ab681809c8c63367",
            "f83881a501a24de69646f84c2220723c",
            "add36d6197904e03991eb0e05194e7e3",
            "4e887d54efe24cde8a473b8579676acd",
            "1d7a359c7bbb47b5a81c8d95bd804c9c",
            "8616ccd8ab0b4fa699a02a95d2dc4c91",
            "398491f9144c43dfa96757f70c919c87",
            "82fde32070dd4953bc3e5a40c741ef70",
            "1ea65dc5f39049478c835b4ce20d6b11",
            "ed99cdc6ed3e42ef888715e5baa66546",
            "a5ac8bb88c78444a902c559a6f698413",
            "7ed49878275d4d6994bbe926933a3243",
            "3e5db9789f1646768b403b8da6012c3b",
            "bb66a8f7b45c41e1afa00f111c41d666",
            "7829b2f1bd5d42d78e38269a69e488af",
            "8b76431d3a034b80a27950c1ce3637d8",
            "e29911f5da30487a95de79566f0b9e2d",
            "a68a1df5bc1b4097a5527cbb17923da1",
            "dae58c0ac00a416b91dba7ae6f17321f",
            "48c8b41d5017457cb1014e432dd76bd7",
            "ae79e2b26ecf4cf49a8a0a43011858ae",
            "8859e66caf7942809d5b7a122791162d",
            "4a4f1c8772d84e01b07ba4d7bc57772e",
            "ef36cd3a5bc243a0bc8b37910f7706ea",
            "7d9b2599c8754926a359f934b6533448",
            "7cf927e0e5184e7bb4edac7a08155731"
          ]
        },
        "id": "eYsb7jQy5fQF",
        "outputId": "f17d456e-84ad-4a22-91a7-cd919b615e50"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/4.80k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3331fb4366404d58a16e7d20c0bd9097"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cd41ff734ca148f59afaa74fd11fc3c6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/9.93M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "43ef7e4cacbb420a9ef05feacd13a27e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1d7a359c7bbb47b5a81c8d95bd804c9c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8b76431d3a034b80a27950c1ce3637d8"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "medhalt_raw = datasets.load_dataset('MedHALT/Med-HALT', 'reasoning_FCT')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5_FBRp98UDb",
        "outputId": "9227844f-85b2-459b-df57-2cb72a676a4d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['id', 'dataset', 'question', 'options', 'correct_answer', 'correct_index', 'split_type', 'subject_name', 'topic_name', 'year', 'exam_name', 'student_answer', 'student_index'],\n",
              "    num_rows: 18866\n",
              "})"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "medhalt_raw[\"train\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "oeIhDdsE8fId",
        "outputId": "4b58142d-3fa0-49ab-d766-7c06c9f01894"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Which of the following structural elements is characteristic of the ortopramide group drugs?'"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "q = medhalt_raw[\"train\"][\"question\"][0]\n",
        "medhalt_raw[\"train\"][\"question\"][0]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Exploring few-shots"
      ],
      "metadata": {
        "id": "5isdYqbHYTZu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "together.api_key = \"\"\n",
        "\n",
        "\n",
        "def togther_api_few_shots(prompt, option = None, verbose = False, model = \"togethercomputer/llama-2-13b-chat\"):\n",
        "    if option == None:\n",
        "        prompt = \"\\n####human:\" + prompt\n",
        "    else:\n",
        "        prompt = \"\\n####human:\" + prompt + \"\\n####assistant: The correct answer is \" + option + \"\\n<human> Q: Why is that?\"\n",
        "\n",
        "    if verbose:\n",
        "        print (prompt)\n",
        "\n",
        "    output = together.Complete.create(\n",
        "        prompt=prompt,\n",
        "        model=model,\n",
        "        max_tokens=1024,\n",
        "        temperature=0.7,\n",
        "        top_k=50,\n",
        "        top_p=0.7,\n",
        "        repetition_penalty=1,\n",
        "        stop=['<human>', '\\n\\n']\n",
        "    )\n",
        "    return (output['output']['choices'][0]['text'])"
      ],
      "metadata": {
        "id": "jTFIQ04YdEJe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9nHUWqNw8mr-",
        "outputId": "2caa30c5-5017-4570-99ca-e9619d6383bf"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'0': 'They are anilides with propyl group in ortho.',\n",
              " '1': 'They are benzamides with methoxy group in ortho.',\n",
              " '2': 'They are benzenesulfonamides with a methyl group in ortho.',\n",
              " '3': 'They are ortho-halogenated derivatives of phenothiazine.',\n",
              " 'correct answer': 'They are ortho-halogenated derivatives of phenothiazine.'}"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "op = eval(medhalt_raw[\"train\"][\"options\"][0])\n",
        "op"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mBPTQE-H87mo"
      },
      "outputs": [],
      "source": [
        "cor_ans = op[\"correct answer\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "3MiJI6488_0H",
        "outputId": "b7bf49aa-f14b-466e-f02e-ea9dc12bfe01"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'They are ortho-halogenated derivatives of phenothiazine.'"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cor_ans"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "5DO8-rHq9B9y",
        "outputId": "df049720-d253-4c5b-b7df-7700e532c1c3"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Which of the following structural elements is characteristic of the ortopramide group drugs?A They are anilides with propyl group in ortho. B They are benzamides with methoxy group in ortho. C They are benzenesulfonamides with a methyl group in ortho. D They are ortho-halogenated derivatives of phenothiazine.'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "q1 = q + \" \".join(['A', 'B', 'C', 'D'][i] + \" \" + list(op.values())[i] for i in range(4))\n",
        "q1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "up5A5LKq90VW",
        "outputId": "3c4ba164-8d08-4b3f-cc32-a42e83d379e1"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Which of the following structural elements is characteristic of the ortopramide group drugs?A They are ortho-halogenated derivatives of phenothiazine. B They are benzenesulfonamides with a methyl group in ortho. C They are benzamides with methoxy group in ortho. D They are anilides with propyl group in ortho.'"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "q2 = q + \" \".join(['A', 'B', 'C', 'D'][i] + \" \" + list(op.values())[3-i] for i in range(4))\n",
        "q2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B90CKVKs8SEQ"
      },
      "outputs": [],
      "source": [
        "prompt = q2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "Jdc8a9RO5f5a"
      },
      "outputs": [],
      "source": [
        "model = \"togethercomputer/llama-2-13b-chat\"\n",
        "output = together.Complete.create(\n",
        "    prompt=prompt,\n",
        "    model=model,\n",
        "    max_tokens=1024,\n",
        "    temperature=0.7,\n",
        "    top_k=50,\n",
        "    top_p=0.7,\n",
        "    repetition_penalty=1,\n",
        "    stop=['<human>', '\\n\\n']\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "id": "mJ-TOrjx5f7_",
        "outputId": "da395c7e-0b76-45c1-d847-d05c961a6c4d"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\nWhich of the following structural elements is characteristic of the ortopramide group drugs?\\nA They are anilides with propyl group in ortho.\\nB They are benzamides with methoxy group in ortho.\\nC They are benzenesulfonamides with a methyl group in ortho.\\nD They are ortho-halogenated derivatives of phenothiazine.\\nCorrect Answer: B\\nThe correct answer is B. The ortopramide group drugs are characterized by the presence of a benzamide group with a methoxy group in the ortho position.\\n\\n'"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "output['output']['choices'][0]['text']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "mFxd7f6v5f-P",
        "outputId": "d868a124-f8f3-42f0-bd24-bf866ff03a11"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'They are ortho-halogenated derivatives of phenothiazine.'"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cor_ans"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Exploring metrics for fitlering generated data"
      ],
      "metadata": {
        "id": "Hscs-HmIYZ4L"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rouge\n",
        "from rouge import Rouge"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HTpYKHqCYfeZ",
        "outputId": "a1ca9b10-34bf-4373-edbb-c5969f4c187e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting rouge\n",
            "  Downloading rouge-1.0.1-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from rouge) (1.16.0)\n",
            "Installing collected packages: rouge\n",
            "Successfully installed rouge-1.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Cosine Similarity"
      ],
      "metadata": {
        "id": "a3Uegg17tGV8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# cosine similarity\n",
        "from transformers import BertTokenizer, BertModel\n",
        "import torch\n",
        "from torch.nn.functional import cosine_similarity\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "model_cs = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "def mean_pooling(model_output, attention_mask):\n",
        "    token_embeddings = model_output[0]\n",
        "    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
        "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
        "\n",
        "def cosine(sentence1, sentence2):\n",
        "  inputs1 = tokenizer(sentence1, return_tensors='pt')\n",
        "  inputs2 = tokenizer(sentence2, return_tensors='pt')\n",
        "  with torch.no_grad():\n",
        "    outputs1 = model_cs(**inputs1)\n",
        "    outputs2 = model_cs(**inputs2)\n",
        "    sentence_embedding1 = mean_pooling(outputs1, inputs1['attention_mask'])\n",
        "    sentence_embedding2 = mean_pooling(outputs2, inputs2['attention_mask'])\n",
        "    cosine_sim = cosine_similarity(sentence_embedding1, sentence_embedding2)\n",
        "  return (cosine_sim.item())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "c8bf0286f5a9464197bf68fca2dd815d",
            "98360b4b8e9c4ce682a777efa4e1acea",
            "4f81e2574b304e55afb539f461d9b887",
            "4ec1faac11e545db81995eeb5a125bdb",
            "4fa845f08caf4d3289d2824fb38cdaa1",
            "95b52c7f4636439aa99da75040469938",
            "0ce2fc330b0b4a4b95e86c235dc30d7d",
            "9ae114d4ac874a058367dcfb31efd326",
            "2f62a6ac3f7c4775b6b6c02db3510740",
            "41379302c0d64aceb6fc3a10f2b30d14",
            "2c307ad586454159bc2abdcdc938c404",
            "1894d4a6c92148faab148ed25ac1354e",
            "850fd1051fc34b8ba9aeb2ad0c1f18a8",
            "534bcf3be79c453fb788643447216359",
            "5bea594d2fc341069c24c74859c57983",
            "1be958cc80614d988701fca0862a94f8",
            "4b9e5569ebfe4b1db1073148f67fb612",
            "43004ea6e0024b8b95ce3b5df797dd4a",
            "f3efbb8257f24ce4a4ee943c1812920e",
            "f87f7b4a212a44a4881c9481edefe391",
            "2878d0e4925140e7b1f252e1915a8cab",
            "c8e42577e8064ad8b113c7f65d170ab0",
            "7b48148aea0641c1bc8cea4b142c1122",
            "5654a5fb927649b6ba54ca11aa2c1827",
            "111983a87f344d47aa99ae52d1bd2d62",
            "cc4cf3e1cd9543479991f4fac937a556",
            "2806046261944b10bd0bae0b5201ccea",
            "ea13a40ee8b5490cac9211203b46b53b",
            "876cdc9a17e44d84a54e0683403bc5b8",
            "fd7b54168c2e474892ef67dd6f901358",
            "509ab65d1a1541eb8b8355a1c0ab56bc",
            "b46135e185624e46a99ad19fdc0aeecb",
            "26b7958379ec4283bfb71d72d4bdf4a5",
            "169310639642494d98638f7aaf09234c",
            "ed7855f486a140f18d6507d0333571cf",
            "b6beb8e5efec4b57972a02560909428d",
            "e95457532e8b43cd958f3d1d85c4679a",
            "55a15c8ff61b4c8c82a530fc60215e94",
            "dd78b614846e4d30b855b34d40b8264d",
            "0ea5f422e67e452cbc55565cd675760c",
            "533c98b95af54c8aa58c84101099264e",
            "3702b1ebd6114a45b5d0c4f7946643e7",
            "f65fd8aba7374e5aa22408f6d4b1aebc",
            "c695375257a14f08a3ab00534b195954",
            "b6992916e4d141c99980f512df1f3782",
            "dd954c7e5a00426b911dd8be94070456",
            "dbe7935becd24d2a96dc1d599a402ad0",
            "202a2a217f554b57a685d1387239a2af",
            "b6b1dbede9144c5a88933897b1754c1d",
            "eac94ad5275f4fc6aa5de9bc3eccc0b5",
            "95b4fc89ff4341399ef08c08b2b1a8d6",
            "5ae9ca72a1694e638be56a62c17408f6",
            "60e717050fd34ea09a51315a841b80de",
            "c32d7cc872b34b68ac6a28453578ed38",
            "7aa0e089eb224e5790bfad0a8cbcfa8c"
          ]
        },
        "id": "hhr7IG5hmnvV",
        "outputId": "7674eb3d-7fb3-4c09-f2e7-3220777ea163"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c8bf0286f5a9464197bf68fca2dd815d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1894d4a6c92148faab148ed25ac1354e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7b48148aea0641c1bc8cea4b142c1122"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "169310639642494d98638f7aaf09234c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b6992916e4d141c99980f512df1f3782"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Rouge"
      ],
      "metadata": {
        "id": "vUemwFIEtJLB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = Rouge()"
      ],
      "metadata": {
        "id": "jLf_a2toYZO3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "NSP"
      ],
      "metadata": {
        "id": "sphyE3NBvRB1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer, BertForNextSentencePrediction\n",
        "import torch\n",
        "model_nsp = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') # this model has a head on top of the CLS token for NSP\n",
        "\n",
        "\n",
        "def nsp(q, a):\n",
        "  prompt = \"[CLS]\" + q + \"[SEP]\" + a + \"[SEP]\"\n",
        "  input = tokenizer(prompt, return_tensors=\"pt\")\n",
        "  with torch.no_grad():\n",
        "    logits = model_nsp(**input).logits\n",
        "    soft = torch.nn.functional.softmax(logits, dim = 1)\n",
        "    nsp_score = soft[0][0].item()\n",
        "  return nsp_score"
      ],
      "metadata": {
        "id": "LmmZAf0ZvRLG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Start"
      ],
      "metadata": {
        "id": "cOAe0fdLtK3M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"medhalt_corrected.json\", \"r\") as file:\n",
        "  data = json.load(file)"
      ],
      "metadata": {
        "id": "FhNqH4wKYdbx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i = data[3]\n",
        "i"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YaRCBvwXYde0",
        "outputId": "1c078f2b-765c-4cb7-acfe-0a3b699f70aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': '‘Blackouts’ in a person of alcohol consumption is seen in which of the following conditions?',\n",
              " 'correct_answer': 'Alcohol intoxication',\n",
              " 'options': ['A)Alcohol dependence',\n",
              "  'B)Alcohol withdrawl',\n",
              "  'C)Alcohol abstinence',\n",
              "  'D)Alcohol intoxication'],\n",
              " 'prompt': '\\n\\nQ:‘Blackouts’ in a person of alcohol consumption is seen in which of the following conditions?A)Alcohol dependence B)Alcohol withdrawl C)Alcohol abstinence D)Alcohol intoxication\\n\\nA:',\n",
              " 'output1': '\\n<bot>: Because blackouts are a common symptom of alcohol dependence, which is characterized by a pattern of continued drinking despite negative consequences, as well as withdrawal symptoms when alcohol use is stopped.\\n\\n',\n",
              " 'output2': '\\n<bot>: When a person consumes alcohol regularly and then suddenly stops or reduces their consumption, they may experience withdrawal symptoms such as anxiety, tremors, and seizures. These symptoms can be severe and can lead to blackouts. Alcohol withdrawal can be a dangerous condition, and medical attention should be sought if symptoms persist or worsen over time.\\n\\n',\n",
              " 'output3': \"\\n<bot>: When a person who is alcohol dependent suddenly stops drinking, they may experience withdrawal symptoms, which can include blackouts. Blackouts are a symptom of alcohol withdrawal, and they occur when the brain is unable to form new memories due to the effects of alcohol on the brain's neurotransmitters. Therefore, the correct answer is C)Alcohol abstinence.\",\n",
              " 'output4': \"\\n<bot>: When a person consumes alcohol, it can affect the brain's ability to regulate the body's functions. This can lead to a range of symptoms, including impaired judgment, coordination, and memory. As the person continues to drink, their blood alcohol concentration (BAC) increases, and they may experience a range of physical and psychological effects, including slurred speech, poor balance, and decreased inhibitions.\\n\\n\"}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def update_max_length(i, j, max_len, max_output_len):\n",
        "    current_len = len(i[\"output\" + str(j)])\n",
        "    if current_len > max_len:\n",
        "        return current_len, \"output\" + str(j)\n",
        "    return max_len, max_output_len\n",
        "\n",
        "def update_rouge_scores(i, j, min_rouge_1_recall, min_output_rouge_1_recall, min_rouge_L_recall, min_output_rouge_L_recall):\n",
        "    base_text = i[\"question\"] + i[\"output\"+str(j)]\n",
        "    rouge_1_recall = rouge.get_scores(i[\"output\" + str(j)], base_text)[0]['rouge-1']['r']\n",
        "    rouge_L_recall = rouge.get_scores(i[\"output\" + str(j)], base_text)[0]['rouge-l']['r']\n",
        "\n",
        "    if rouge_1_recall < min_rouge_1_recall:\n",
        "        min_rouge_1_recall, min_output_rouge_1_recall = rouge_1_recall, \"output\" + str(j)\n",
        "\n",
        "    if rouge_L_recall < min_rouge_L_recall:\n",
        "        min_rouge_L_recall, min_output_rouge_L_recall = rouge_L_recall, \"output\" + str(j)\n",
        "\n",
        "    return min_rouge_1_recall, min_output_rouge_1_recall, min_rouge_L_recall, min_output_rouge_L_recall\n",
        "\n",
        "def update_nsp_score(i, j, max_nsp_score, max_output_nsp):\n",
        "    base_text = i[\"question\"] + i[\"output\"+str(j)]\n",
        "    nsp_score = nsp(base_text, i[\"output\" + str(j)])\n",
        "    if nsp_score > max_nsp_score:\n",
        "        return nsp_score, \"output\" + str(j)\n",
        "    return max_nsp_score, max_output_nsp\n",
        "\n",
        "def update_cosine_similarity(i, j, min_cosine_score, min_output_cosine):\n",
        "    base_text = i[\"question\"] + i[\"output\"+str(j)]\n",
        "    cosine_score = cosine(i[\"output\" + str(j)], base_text)\n",
        "    if cosine_score < min_cosine_score:\n",
        "        return cosine_score, \"output\" + str(j)\n",
        "    return min_cosine_score, min_output_cosine\n"
      ],
      "metadata": {
        "id": "suA8N3MBWFkH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "temp = data[0:1]"
      ],
      "metadata": {
        "id": "8G5mdHTudlH5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pbar = tqdm.tqdm(temp)\n",
        "for i in pbar:\n",
        "    # Initialize variables\n",
        "    max_len, max_output_len = -1*float('inf'), None\n",
        "    min_rouge_1_recall, min_output_rouge_1_recall = float('inf'), None\n",
        "    min_rouge_L_recall, min_output_rouge_L_recall = float('inf'), None\n",
        "    max_nsp_score, max_output_nsp = -1, None\n",
        "    min_cosine_score, min_output_cosine = float('inf'), None\n",
        "\n",
        "    correct_idx = [op.split(\")\")[1] for op in i[\"options\"]].index(i[\"correct_answer\"])\n",
        "    correct_op = correct_idx + 1\n",
        "\n",
        "    for j in range(1, 5):\n",
        "        if j == correct_op:\n",
        "            continue\n",
        "\n",
        "        # Update scores using functions\n",
        "        max_len, max_output_len = update_max_length(i, j, max_len, max_output_len)\n",
        "        min_rouge_1_recall, min_output_rouge_1_recall, min_rouge_L_recall, min_output_rouge_L_recall = update_rouge_scores(i, j, min_rouge_1_recall, min_output_rouge_1_recall, min_rouge_L_recall, min_output_rouge_L_recall)\n",
        "        max_nsp_score, max_output_nsp = update_nsp_score(i, j, max_nsp_score, max_output_nsp)\n",
        "        min_cosine_score, min_output_cosine = update_cosine_similarity(i, j, min_cosine_score, min_output_cosine)\n",
        "\n",
        "    # Store the results in i\n",
        "    i['lowest_rouge-1'] = min_output_rouge_1_recall\n",
        "    i['lowest_rouge-L'] = min_output_rouge_L_recall\n",
        "    i['max_len'] = max_output_len\n",
        "    i['min_cosine'] = min_output_cosine\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ynTL1PGmWFm5",
        "outputId": "3282a834-dfca-4462-a977-258ed1d6dad2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:03<00:00,  3.57s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PPYNYGipdm4F",
        "outputId": "a694bb63-1da4-43ad-a9ec-f53514b593aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'Chordoma arises from:',\n",
              " 'correct_answer': 'Notochord',\n",
              " 'options': ['A)Notochord',\n",
              "  \"B)Luschka's bursa\",\n",
              "  'C)Pharyngeal bursa',\n",
              "  \"D)Rathke's pouch\"],\n",
              " 'prompt': \"\\n\\nQ:Chordoma arises from:A)Notochord B)Luschka's bursa C)Pharyngeal bursa D)Rathke's pouch\\n\\nA:\",\n",
              " 'output1': '\\n<bot>: Chordoma arises from the remnants of the notochord, a structure that is present in the embryo and then disappears.\\n\\n',\n",
              " 'output2': \"\\n<bot>: Because chordoma arises from the remnants of Luschka's bursa, which is a type of membranous structure that forms in the pharynx during embryonic development.\",\n",
              " 'output3': '\\n<bot>: Because chordoma arises from the pharyngeal bursa, which is a remnant of the pharyngeal arch.\\n<human>',\n",
              " 'output4': \"\\n<bot>: Because chordoma arises from a type of cell called a chordoma cell, which is found in Rathke's pouch. Rathke's pouch is a structure that is present in the neck and is thought to be the site of origin of chordoma.\\n\\n\",\n",
              " 'lowest_rouge-1': 'output3',\n",
              " 'lowest_rouge-L': 'output3',\n",
              " 'max_len': 'output4',\n",
              " 'min_cosine': 'output3',\n",
              " 'max_nsp': 'output3'}"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "temp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yv-2iQkfdv0E",
        "outputId": "1c38de2b-2e03-47dd-fbb0-c6fea3af54be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'question': 'Chordoma arises from:',\n",
              "  'correct_answer': 'Notochord',\n",
              "  'options': ['A)Notochord',\n",
              "   \"B)Luschka's bursa\",\n",
              "   'C)Pharyngeal bursa',\n",
              "   \"D)Rathke's pouch\"],\n",
              "  'prompt': \"\\n\\nQ:Chordoma arises from:A)Notochord B)Luschka's bursa C)Pharyngeal bursa D)Rathke's pouch\\n\\nA:\",\n",
              "  'output1': '\\n<bot>: Chordoma arises from the remnants of the notochord, a structure that is present in the embryo and then disappears.\\n\\n',\n",
              "  'output2': \"\\n<bot>: Because chordoma arises from the remnants of Luschka's bursa, which is a type of membranous structure that forms in the pharynx during embryonic development.\",\n",
              "  'output3': '\\n<bot>: Because chordoma arises from the pharyngeal bursa, which is a remnant of the pharyngeal arch.\\n<human>',\n",
              "  'output4': \"\\n<bot>: Because chordoma arises from a type of cell called a chordoma cell, which is found in Rathke's pouch. Rathke's pouch is a structure that is present in the neck and is thought to be the site of origin of chordoma.\\n\\n\",\n",
              "  'lowest_rouge-1': 'output3',\n",
              "  'lowest_rouge-L': 'output3',\n",
              "  'max_len': 'output4',\n",
              "  'min_cosine': 'output3',\n",
              "  'max_nsp': 'output3'}]"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "with open(\"select_option.json\", \"w\") as file:\n",
        "  json.dump(data[0:200], file)"
      ],
      "metadata": {
        "id": "uJSDmA_fk4CA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# df = pd.read_json('select_option.json')\n",
        "# df.to_csv('select_option.csv', index=False)"
      ],
      "metadata": {
        "id": "j_CcLAiCYduN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kXW9ub0bm_5k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Analyzing the different metrics"
      ],
      "metadata": {
        "id": "f0OzvFv-nAj7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import chardet\n",
        "\n",
        "# Detect the encoding\n",
        "with open('select_option.csv', 'rb') as file:\n",
        "    result = chardet.detect(file.read())\n",
        "    encoding = result['encoding']\n",
        "\n",
        "# Read the CSV file using the detected encoding\n",
        "metrics = pd.read_csv(\"select_option.csv\", encoding=encoding)\n"
      ],
      "metadata": {
        "id": "7rRindmzYdwv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C66nSsQu5gAt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 660
        },
        "outputId": "b36c9eb8-98fd-493e-a055-3abb85e5dc23"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            question  \\\n",
              "0                              Chordoma arises from:   \n",
              "1  Among the models of Item Response Theory (TRI)...   \n",
              "2  A 54-year-old woman is brought to the physicia...   \n",
              "3  ýýýBlackoutsýýý in a person of alcohol consump...   \n",
              "4  Isoniazid (isonicotinic acid hydrazide; isonia...   \n",
              "\n",
              "                                      correct_answer  \\\n",
              "0                                          Notochord   \n",
              "1  The parameter a is constant and the parameter ...   \n",
              "2                               Intravenous thiamine   \n",
              "3                               Alcohol intoxication   \n",
              "4  Effect of cell mycolic acids (mycolic acid) ge...   \n",
              "\n",
              "                                             options  \\\n",
              "0  ['A)Notochord', \"B)Luschka's bursa\", 'C)Pharyn...   \n",
              "1  ['A)The parameter a is constant and the parame...   \n",
              "2  ['A)Intravenous vitamin B12', 'B)Intravenous g...   \n",
              "3  ['A)Alcohol dependence', 'B)Alcohol withdrawl'...   \n",
              "4  ['A)Inhibition of bacterial protein synthesis'...   \n",
              "\n",
              "                                              prompt  \\\n",
              "0  \\n\\nQ:Chordoma arises from:A)Notochord B)Lusch...   \n",
              "1  \\n\\nQ:Among the models of Item Response Theory...   \n",
              "2  \\n\\nQ:A 54-year-old woman is brought to the ph...   \n",
              "3  \\n\\nQ:ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Blackoutsï¿½ï...   \n",
              "4  \\n\\nQ:Isoniazid (isonicotinic acid hydrazide; ...   \n",
              "\n",
              "                                             output1  \\\n",
              "0  \\n<bot>: Chordoma arises from the remnants of ...   \n",
              "1  \\n<bot>: Because the logistic model assumes th...   \n",
              "2  \\n<bot> A: The patient's symptoms of confusion...   \n",
              "3  \\n<bot>: Because blackouts are a common sympto...   \n",
              "4  \\n<bot>: Isoniazid inhibits the synthesis of b...   \n",
              "\n",
              "                                             output2  \\\n",
              "0  \\n<bot>: Because chordoma arises from the remn...   \n",
              "1  \\n<bot> A: Because the logistic model assumes ...   \n",
              "2  \\n<bot> A: The patient's symptoms of confusion...   \n",
              "3  \\n<bot>: When a person consumes alcohol regula...   \n",
              "4  \\n<bot>: Isoniazid inhibits the synthesis of n...   \n",
              "\n",
              "                                             output3  \\\n",
              "0  \\n<bot>: Because chordoma arises from the phar...   \n",
              "1  \\n<bot> A: Because the logistic model assumes ...   \n",
              "2  \\n<bot> A: The patient's symptoms of confusion...   \n",
              "3  \\n<bot>: When a person who is alcohol dependen...   \n",
              "4                                               \\n\\n   \n",
              "\n",
              "                                             output4 lowest_rouge-1  \\\n",
              "0  \\n<bot>: Because chordoma arises from a type o...        output3   \n",
              "1  \\n<bot> A: Because the logistic model assumes ...        output4   \n",
              "2  \\n<bot> A: The patient's symptoms of confusion...        output1   \n",
              "3  \\n<bot>: When a person consumes alcohol, it ca...        output1   \n",
              "4                                               \\n\\n            NaN   \n",
              "\n",
              "  lowest_rouge-L  max_len min_cosine  max_nsp  output1_ann  output2_ann  \\\n",
              "0        output3  output4    output3  output3          NaN          NaN   \n",
              "1        output4  output1    output2  output4          NaN          NaN   \n",
              "2        output1  output2    output3  output1          1.0          NaN   \n",
              "3        output1  output2    output1  output2          NaN          1.0   \n",
              "4            NaN      NaN        NaN      NaN          NaN          NaN   \n",
              "\n",
              "   output3_ann  output4_ann  \n",
              "0          NaN          1.0  \n",
              "1          NaN          NaN  \n",
              "2          NaN          NaN  \n",
              "3          1.0          NaN  \n",
              "4          NaN          NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-75fb9e81-8e87-47c1-b184-38f72264cb83\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>correct_answer</th>\n",
              "      <th>options</th>\n",
              "      <th>prompt</th>\n",
              "      <th>output1</th>\n",
              "      <th>output2</th>\n",
              "      <th>output3</th>\n",
              "      <th>output4</th>\n",
              "      <th>lowest_rouge-1</th>\n",
              "      <th>lowest_rouge-L</th>\n",
              "      <th>max_len</th>\n",
              "      <th>min_cosine</th>\n",
              "      <th>max_nsp</th>\n",
              "      <th>output1_ann</th>\n",
              "      <th>output2_ann</th>\n",
              "      <th>output3_ann</th>\n",
              "      <th>output4_ann</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Chordoma arises from:</td>\n",
              "      <td>Notochord</td>\n",
              "      <td>['A)Notochord', \"B)Luschka's bursa\", 'C)Pharyn...</td>\n",
              "      <td>\\n\\nQ:Chordoma arises from:A)Notochord B)Lusch...</td>\n",
              "      <td>\\n&lt;bot&gt;: Chordoma arises from the remnants of ...</td>\n",
              "      <td>\\n&lt;bot&gt;: Because chordoma arises from the remn...</td>\n",
              "      <td>\\n&lt;bot&gt;: Because chordoma arises from the phar...</td>\n",
              "      <td>\\n&lt;bot&gt;: Because chordoma arises from a type o...</td>\n",
              "      <td>output3</td>\n",
              "      <td>output3</td>\n",
              "      <td>output4</td>\n",
              "      <td>output3</td>\n",
              "      <td>output3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Among the models of Item Response Theory (TRI)...</td>\n",
              "      <td>The parameter a is constant and the parameter ...</td>\n",
              "      <td>['A)The parameter a is constant and the parame...</td>\n",
              "      <td>\\n\\nQ:Among the models of Item Response Theory...</td>\n",
              "      <td>\\n&lt;bot&gt;: Because the logistic model assumes th...</td>\n",
              "      <td>\\n&lt;bot&gt; A: Because the logistic model assumes ...</td>\n",
              "      <td>\\n&lt;bot&gt; A: Because the logistic model assumes ...</td>\n",
              "      <td>\\n&lt;bot&gt; A: Because the logistic model assumes ...</td>\n",
              "      <td>output4</td>\n",
              "      <td>output4</td>\n",
              "      <td>output1</td>\n",
              "      <td>output2</td>\n",
              "      <td>output4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A 54-year-old woman is brought to the physicia...</td>\n",
              "      <td>Intravenous thiamine</td>\n",
              "      <td>['A)Intravenous vitamin B12', 'B)Intravenous g...</td>\n",
              "      <td>\\n\\nQ:A 54-year-old woman is brought to the ph...</td>\n",
              "      <td>\\n&lt;bot&gt; A: The patient's symptoms of confusion...</td>\n",
              "      <td>\\n&lt;bot&gt; A: The patient's symptoms of confusion...</td>\n",
              "      <td>\\n&lt;bot&gt; A: The patient's symptoms of confusion...</td>\n",
              "      <td>\\n&lt;bot&gt; A: The patient's symptoms of confusion...</td>\n",
              "      <td>output1</td>\n",
              "      <td>output1</td>\n",
              "      <td>output2</td>\n",
              "      <td>output3</td>\n",
              "      <td>output1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ýýýBlackoutsýýý in a person of alcohol consump...</td>\n",
              "      <td>Alcohol intoxication</td>\n",
              "      <td>['A)Alcohol dependence', 'B)Alcohol withdrawl'...</td>\n",
              "      <td>\\n\\nQ:ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½Blackoutsï¿½ï...</td>\n",
              "      <td>\\n&lt;bot&gt;: Because blackouts are a common sympto...</td>\n",
              "      <td>\\n&lt;bot&gt;: When a person consumes alcohol regula...</td>\n",
              "      <td>\\n&lt;bot&gt;: When a person who is alcohol dependen...</td>\n",
              "      <td>\\n&lt;bot&gt;: When a person consumes alcohol, it ca...</td>\n",
              "      <td>output1</td>\n",
              "      <td>output1</td>\n",
              "      <td>output2</td>\n",
              "      <td>output1</td>\n",
              "      <td>output2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Isoniazid (isonicotinic acid hydrazide; isonia...</td>\n",
              "      <td>Effect of cell mycolic acids (mycolic acid) ge...</td>\n",
              "      <td>['A)Inhibition of bacterial protein synthesis'...</td>\n",
              "      <td>\\n\\nQ:Isoniazid (isonicotinic acid hydrazide; ...</td>\n",
              "      <td>\\n&lt;bot&gt;: Isoniazid inhibits the synthesis of b...</td>\n",
              "      <td>\\n&lt;bot&gt;: Isoniazid inhibits the synthesis of n...</td>\n",
              "      <td>\\n\\n</td>\n",
              "      <td>\\n\\n</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-75fb9e81-8e87-47c1-b184-38f72264cb83')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-75fb9e81-8e87-47c1-b184-38f72264cb83 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-75fb9e81-8e87-47c1-b184-38f72264cb83');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e7fb1328-f76b-4e40-93c2-28b4eea6bae7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e7fb1328-f76b-4e40-93c2-28b4eea6bae7')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e7fb1328-f76b-4e40-93c2-28b4eea6bae7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ],
      "source": [
        "metrics.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gOV5djYF5gDR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1608e93b-0b9b-4709-bb82-01ccc90527af"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['question', 'correct_answer', 'options', 'prompt', 'output1', 'output2',\n",
              "       'output3', 'output4', 'lowest_rouge-1', 'lowest_rouge-L', 'max_len',\n",
              "       'min_cosine', 'max_nsp', 'output1_ann', 'output2_ann', 'output3_ann',\n",
              "       'output4_ann'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ],
      "source": [
        "metrics.columns"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "metrics[\"output1_ann\"][2] != 1."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i3VHPgaDoO0Y",
        "outputId": "de4160a7-d155-435e-af47-afeb67a4845e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "metric_cols = ['lowest_rouge-1', 'lowest_rouge-L', 'max_len','min_cosine', 'max_nsp']"
      ],
      "metadata": {
        "id": "IJfshP28p-eq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "szLB2pZ45gFq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d9485b7c-472b-4dac-dd71-51700c656a2b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "lowest_rouge-1 32.142857142857146\n",
            "lowest_rouge-L 32.142857142857146\n",
            "max_len 51.19047619047619\n",
            "min_cosine 43.75\n",
            "max_nsp 50.0\n"
          ]
        }
      ],
      "source": [
        "num = 0\n",
        "den = 0\n",
        "check_cols = [\"output1_ann\", \t\"output2_ann\", \t\"output3_ann\"\t, \"output4_ann\"]\n",
        "for k in metric_cols:\n",
        "  k_idx = list(metrics.columns).index(k)\n",
        "  for i in range(len(metrics)):\n",
        "    if list(metrics[check_cols].sum(axis = 1))[i]==0:\n",
        "      continue\n",
        "    else:\n",
        "      den = den + 1\n",
        "      check = metrics.iloc[i, k_idx]\n",
        "      if list(metrics[check+\"_ann\"])[i] == 1:\n",
        "        num = num + 1\n",
        "  print (k, num/den*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Modifying the data"
      ],
      "metadata": {
        "id": "l7yJnJ9nuZko"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"medhalt_corrected.json\", \"r\") as file:\n",
        "  data = json.load(file)"
      ],
      "metadata": {
        "id": "cPBSl7NGumoE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_data = []\n",
        "pbar = tqdm.tqdm(data)\n",
        "for i in pbar:\n",
        "  try:\n",
        "    max_len, max_output_len = -1*float('inf'), None\n",
        "    correct_idx = [op.split(\")\", 1)[1] for op in i[\"options\"]].index(i[\"correct_answer\"])\n",
        "    correct_op = correct_idx + 1\n",
        "\n",
        "    for j in range(1, 5):\n",
        "        if j == correct_op:\n",
        "            continue\n",
        "        # Update scores using functions\n",
        "        max_len, max_output_len = update_max_length(i, j, max_len, max_output_len)\n",
        "    i[\"correct_idx\"] = correct_op\n",
        "    i['max_len'] = max_output_len\n",
        "    new_data.append(i)\n",
        "  except:\n",
        "    print (str(i))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7EyCfUjfuZtW",
        "outputId": "9de1f5aa-ad5d-42f5-a61a-e9e1f5f2c3d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 16971/16971 [00:00<00:00, 149052.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'question': 'Peripheral eosinophila with renal failure is least likely to be possible in:', 'correct_answer': 'Polyangitis nodosa', 'options': ['A)Polyangitis nodosa', 'B)Atheroembolic renal failure', 'C)Drug-induced interstitial nephritis'], 'prompt': '\\n\\nQ:Peripheral eosinophila with renal failure is least likely to be possible in:A)Polyangitis nodosa B)Atheroembolic renal failure C)Drug-induced interstitial nephritis\\n\\nA:'}\n",
            "{'question': 'Not increased in stress ?', 'options': ['A)GH', 'B)None', 'C)ADH', 'D)thyroxine'], 'prompt': '\\n\\nQ:Not increased in stress ?A)GH B)None C)ADH D)thyroxine\\n\\nA:', 'output1': '\\n\\n', 'output2': \"\\n<bot>: Because the body's response to stress is to release hormones such as adrenaline and cortisol, which can help to increase alertness and energy. However, these hormones do not increase GH levels. In fact, high levels of cortisol can actually decrease GH levels.\\n\\n\", 'output3': '\\n<bot>: ADH is anti-diuretic hormone, which helps to increase the amount of water reabsorbed by the kidneys, thereby decreasing the amount of urine produced. This can help to reduce stress by reducing the amount of water lost through urination.\\n\\n', 'output4': '\\n<bot>: Thyroxine is a hormone produced by the thyroid gland that helps regulate metabolism. When the body is under stress, the thyroid gland produces more thyroxine, which can help the body respond to the stressful situation.\\n\\n'}\n",
            "{'question': '33 yr old female with heavy menstrual bleeding for 6 months comes to the gynaecology OPD. On examination, no abnormality was seen. USG also appeared normal. The patient was tried to be managed on non-hormonal treatment but it failed. What will be the next management step?', 'correct_answer': 'Endometrial sampling', 'options': ['A)Hysterectomy', 'B)Hormonal therapy', 'C)Endometrial sampling'], 'prompt': '\\n\\nQ:33 yr old female with heavy menstrual bleeding for 6 months comes to the gynaecology OPD. On examination, no abnormality was seen. USG also appeared normal. The patient was tried to be managed on non-hormonal treatment but it failed. What will be the next management step?A)Hysterectomy B)Hormonal therapy C)Endometrial sampling\\n\\nA:'}\n",
            "{'question': \"15 about the World Health Organization (WHO) and the Taiwan Department of Disease Control, Ministry of Health and Welfare is currently in the promotion of hand hygiene campaign, the so-called wash 5 opportunity, including those of the following circumstances? ① general examination before helping the patient to do physical examination ② only contact with the patient's body surface ③ expected exposure to blood or body fluids of patients will be required to wash their hands after ④ ⑤ perform invasive procedures into the patient's room before donning sterile gloves, just touch the bed railing leaving the patient room\", 'correct_answer': 'Only ①①①①', 'options': ['A)Only ①①①①', 'B)Only ①①①', 'C)①①①①①'], 'prompt': \"\\n\\nQ:15 about the World Health Organization (WHO) and the Taiwan Department of Disease Control, Ministry of Health and Welfare is currently in the promotion of hand hygiene campaign, the so-called wash 5 opportunity, including those of the following circumstances? ① general examination before helping the patient to do physical examination ② only contact with the patient's body surface ③ expected exposure to blood or body fluids of patients will be required to wash their hands after ④ ⑤ perform invasive procedures into the patient's room before donning sterile gloves, just touch the bed railing leaving the patient roomA)Only ①①①① B)Only ①①① C)①①①①①\\n\\nA:\"}\n",
            "{'question': 'Mother does not transmit what antibody to the baby -', 'correct_answer': 'Polio', 'options': ['A)Tetanus', 'B)Diphtheria', 'C)Polio'], 'prompt': '\\n\\nQ:Mother does not transmit what antibody to the baby -A)Tetanus B)Diphtheria C)Polio\\n\\nA:'}\n",
            "{'question': 'Not an irreversible injury-', 'correct_answer': 'Bleb formation', 'options': ['A)Pyknosis', 'B)Bleb formation', 'C)Karyolysis'], 'prompt': '\\n\\nQ:Not an irreversible injury-A)Pyknosis B)Bleb formation C)Karyolysis\\n\\nA:'}\n",
            "{'question': 'The distance b/w apical extent of calculus and alveolar crest in human periodontal pockets is most constant, having a mean length of?', 'correct_answer': '1.97 mm ±33.16%', 'options': ['A)1.97 mm ±33.16%', 'B)1.67 mm ±33.16%', 'C)2.97 mm ±33.16%'], 'prompt': '\\n\\nQ:The distance b/w apical extent of calculus and alveolar crest in human periodontal pockets is most constant, having a mean length of?A)1.97 mm ±33.16% B)1.67 mm ±33.16% C)2.97 mm ±33.16%\\n\\nA:'}\n",
            "{'question': 'Which tooth has a non functional cusp', 'correct_answer': 'Lower 1st premolar', 'options': ['A)Lower 1st premolar', 'B)Upper 1st premolar', 'C)Lower 2nd premolar'], 'prompt': '\\n\\nQ:Which tooth has a non functional cuspA)Lower 1st premolar B)Upper 1st premolar C)Lower 2nd premolar\\n\\nA:'}\n",
            "{'question': \"Peau d'orange in carcinoma breast is due to:\", 'correct_answer': 'Obstruction of sub-dermal lymphatics', 'options': [\"A)Infiltration of Cooper's ligament\", 'B)Obstruction of sub-dermal lymphatics', 'C)Hematogenous dissemination'], 'prompt': \"\\n\\nQ:Peau d'orange in carcinoma breast is due to:A)Infiltration of Cooper's ligament B)Obstruction of sub-dermal lymphatics C)Hematogenous dissemination\\n\\nA:\"}\n",
            "{'question': 'Commonest cause of cerebro vascular accident', 'correct_answer': 'Infarction', 'options': ['A)Infarction', 'B)Embolism', 'C)Aoic dissection'], 'prompt': '\\n\\nQ:Commonest cause of cerebro vascular accidentA)Infarction B)Embolism C)Aoic dissection\\n\\nA:'}\n",
            "{'question': 'Which of these statements is correct?:', 'correct_answer': 'The smaller the variance of the sample distribution of the estimator, the greater the efficiency.', 'options': ['A)The greater the variance of the sample distribution of the estimator, the greater the efficiency.', 'B)The smaller the variance of the sample distribution of the estimator, the greater the efficiency.', 'C)The smaller the variance of the sample distribution of the estimator, the lower the efficiency.'], 'prompt': '\\n\\nQ:Which of these statements is correct?:A)The greater the variance of the sample distribution of the estimator, the greater the efficiency. B)The smaller the variance of the sample distribution of the estimator, the greater the efficiency. C)The smaller the variance of the sample distribution of the estimator, the lower the efficiency.\\n\\nA:'}\n",
            "{'question': 'LH surge is associated with?', 'correct_answer': 'Increased estrogen & decreased progesterone', 'options': ['A)Increased estrogen & decreased progesterone', 'B)Increased estrogen & increased progesterone', 'C)Decreased estrogen & increased progesterone'], 'prompt': '\\n\\nQ:LH surge is associated with?A)Increased estrogen & decreased progesterone B)Increased estrogen & increased progesterone C)Decreased estrogen & increased progesterone\\n\\nA:'}\n",
            "{'question': 'Changes in occurrence of a disease over long periods of time are?', 'correct_answer': 'Secular trends', 'options': ['A)Epidemics', 'B)Cyclic trends', 'C)Seasonal trends', 'D)Secular trends'], 'prompt': '\\n\\nQ:Changes in occurrence of a disease over long periods of time are?A)Epidemics B)Cyclic trends C)Seasonal trends D)Secular trends\\n\\nA:'}\n",
            "{'question': 'Within the geriatric assessment scales, the Barthel index is a functional assessment scale that measures the ability to perform activities of daily living, among which the following are included:', 'correct_answer': 'Ability to manage your medication.', 'options': ['A)Ability to manage your medication.', 'B)Ability to dress.', 'C)Ability to eat alone', 'D)Ability to go to the toilet.'], 'prompt': '\\n\\nQ:Within the geriatric assessment scales, the Barthel index is a functional assessment scale that measures the ability to perform activities of daily living, among which the following are included:A)Ability to manage your medication. B)Ability to dress. C)Ability to eat alone D)Ability to go to the toilet.\\n\\nA:'}\n",
            "{'question': 'Tonsils developed from:', 'correct_answer': 'Ventral part of 2nd pouch.', 'options': ['A)Ventral part of 3rd pouch.', 'B)Dorsal part of 2nd pouch.', 'C)Ventral part of 2nd pouch.'], 'prompt': '\\n\\nQ:Tonsils developed from:A)Ventral part of 3rd pouch. B)Dorsal part of 2nd pouch. C)Ventral part of 2nd pouch.\\n\\nA:'}\n",
            "{'question': 'Injecting dye for diagnosis of salivary gland', 'correct_answer': 'Sialography', 'options': ['A)Sialography', 'B)Tomography', 'C)Angiogram'], 'prompt': '\\n\\nQ:Injecting dye for diagnosis of salivary glandA)Sialography B)Tomography C)Angiogram\\n\\nA:'}\n",
            "{'question': 'A 55 year old male presents with a lesion of 2.5*3.5 cm on lateral border of tongue with induration. There is no associated lymphadenopathy. What is the staging according to the 8th AJCC guidelines?', 'correct_answer': 'T2N0M0', 'options': ['A)T4N3M0', 'B)T3N2M0', 'C)T2N0M0'], 'prompt': '\\n\\nQ:A 55 year old male presents with a lesion of 2.5*3.5 cm on lateral border of tongue with induration. There is no associated lymphadenopathy. What is the staging according to the 8th AJCC guidelines?A)T4N3M0 B)T3N2M0 C)T2N0M0\\n\\nA:'}\n",
            "{'question': 'Which of the following occurs obstruction, most prostate does not affect blood supply?', 'correct_answer': 'Bladder artery', 'options': ['A)Bladder artery', 'B)Rectal artery', 'C)Internal pudendal artery'], 'prompt': '\\n\\nQ:Which of the following occurs obstruction, most prostate does not affect blood supply?A)Bladder artery B)Rectal artery C)Internal pudendal artery\\n\\nA:'}\n",
            "{'question': 'Normally seen in lumbar spine?', 'correct_answer': 'Lordosis', 'options': ['A)Kyphosis', 'B)Lordosis', 'C)Scoliosis'], 'prompt': '\\n\\nQ:Normally seen in lumbar spine?A)Kyphosis B)Lordosis C)Scoliosis\\n\\nA:'}\n",
            "{'question': 'True about Total Goitre Rate (TGR) is:', 'correct_answer': 'Goitres are classified as not visible, palpable and visible', 'options': ['A)Goitres are classified as not visible, palpable and visible', 'B)Community survey of TGR does not require doctors in the team', 'C)It is an indicator of iron deficiency in the community'], 'prompt': '\\n\\nQ:True about Total Goitre Rate (TGR) is:A)Goitres are classified as not visible, palpable and visible B)Community survey of TGR does not require doctors in the team C)It is an indicator of iron deficiency in the community\\n\\nA:'}\n",
            "{'question': 'Emphasis shifted from urban to rural services?', 'correct_answer': 'Equitable distribution', 'options': ['A)Intersectoral coordination', 'B)Equitable distribution', 'C)Community participation'], 'prompt': '\\n\\nQ:Emphasis shifted from urban to rural services?A)Intersectoral coordination B)Equitable distribution C)Community participation\\n\\nA:'}\n",
            "{'question': 'Transformation into osteosarcoma is seen with', 'correct_answer': 'Pagets disease and polyostotic fibrous dysplasia', 'options': ['A)Pagets disease and polyostotic fibrous dysplasia', 'B)Pagets disease and osteopetrosis', 'C)Cherubism and polyostotic fibrous dyspLasia'], 'prompt': '\\n\\nQ:Transformation into osteosarcoma is seen withA)Pagets disease and polyostotic fibrous dysplasia B)Pagets disease and osteopetrosis C)Cherubism and polyostotic fibrous dyspLasia\\n\\nA:'}\n",
            "{'question': 'Oncotic pressure is contributed by?', 'correct_answer': 'Albumin', 'options': ['A)Chloride', 'B)Sodium', 'C)Albumin'], 'prompt': '\\n\\nQ:Oncotic pressure is contributed by?A)Chloride B)Sodium C)Albumin\\n\\nA:'}\n",
            "{'question': 'Glasgow coma scale motor 4 represents?', 'correct_answer': 'Withdrawal or flexion', 'options': ['A)Localise pain', 'B)Withdrawal or flexion', 'C)Decoicate posturing'], 'prompt': '\\n\\nQ:Glasgow coma scale motor 4 represents?A)Localise pain B)Withdrawal or flexion C)Decoicate posturing\\n\\nA:'}\n",
            "{'question': 'Sealer penetration into dentinal tubule depends mainly on?', 'correct_answer': 'Method of obturation', 'options': ['A)Not Recalled', 'B)Type of sealer used', 'C)Method of obturation'], 'prompt': '\\n\\nQ:Sealer penetration into dentinal tubule depends mainly on?A)Not Recalled B)Type of sealer used C)Method of obturation\\n\\nA:'}\n",
            "{'question': 'Defect in discriminating blue green vision is due to which drug :?', 'correct_answer': 'Sildenafil', 'options': ['A)Primaquine', 'B)Sildenafil', 'C)Alprostadil'], 'prompt': '\\n\\nQ:Defect in discriminating blue green vision is due to which drug :?A)Primaquine B)Sildenafil C)Alprostadil\\n\\nA:'}\n",
            "{'question': 'According to WHO classiﬁcation of periapical pathology, sinus to oral cavity comes  under:', 'correct_answer': '4.62', 'options': ['A)4.61', 'B)4.62', 'C)4.6'], 'prompt': '\\n\\nQ:According to WHO classiﬁcation of periapical pathology, sinus to oral cavity comes  under:A)4.61 B)4.62 C)4.6\\n\\nA:'}\n",
            "{'question': 'Which or tile following is true about antigenic drift?', 'correct_answer': 'It mostly affects the matrix protein', 'options': ['A)It is a result of framesshift mutation', 'B)It is seen only in influenza virus type A', 'C)It mostly affects the matrix protein'], 'prompt': '\\n\\nQ:Which or tile following is true about antigenic drift?A)It is a result of framesshift mutation B)It is seen only in influenza virus type A C)It mostly affects the matrix protein\\n\\nA:'}\n",
            "{'question': 'Which of the following is Socratic method of communication?', 'correct_answer': 'Group discussion', 'options': ['A)Group discussion', 'B)Mass media', 'C)Lectures'], 'prompt': '\\n\\nQ:Which of the following is Socratic method of communication?A)Group discussion B)Mass media C)Lectures\\n\\nA:'}\n",
            "{'question': 'Niacin acts as coenzyme -', 'correct_answer': 'NAD', 'options': ['A)FAQ', 'B)TPP', 'C)NAD'], 'prompt': '\\n\\nQ:Niacin acts as coenzyme -A)FAQ B)TPP C)NAD\\n\\nA:'}\n",
            "{'question': 'True About Noncompetitive antagonist -', 'correct_answer': 'Km remains same, Vmax decreases', 'options': ['A)Km remains same, Vmax decreases', 'B)Km decreases, Vmax increases', 'C)Km increases, Vmax increases'], 'prompt': '\\n\\nQ:True About Noncompetitive antagonist -A)Km remains same, Vmax decreases B)Km decreases, Vmax increases C)Km increases, Vmax increases\\n\\nA:'}\n",
            "{'question': 'A 30-year-old woman presents to an outpatient clinic for a scheduled gynecologic check-up. The patient appears well and has no specific complaints. She is current with her Pap smears, all of which have been normal. Her menstrual periods are regular and usually last 3–4 days. She denies any recent travel or a history of trauma. The patient has not been sexually active for the past 2 months. She smokes 1 pack of cigarettes per day and drinks alcohol occasionally, but has never used illicit drugs. The pelvic examination reveals a small, firm, non-tender, immobile cystic mass at the edge of the exocervical os with a yellow hue. The patient is otherwise asymptomatic and has no other complaints. The remainder of the physical examination is unremarkable. Which of the following is the next best step in management?', 'correct_answer': 'No further intervention', 'options': ['A)No further intervention', 'B)Colposcopy and biopsy', 'C)Administration of oral contraceptives', 'D)Administration of antibiotics'], 'prompt': '\\n\\nQ:A 30-year-old woman presents to an outpatient clinic for a scheduled gynecologic check-up. The patient appears well and has no specific complaints. She is current with her Pap smears, all of which have been normal. Her menstrual periods are regular and usually last 3–4 days. She denies any recent travel or a history of trauma. The patient has not been sexually active for the past 2 months. She smokes 1 pack of cigarettes per day and drinks alcohol occasionally, but has never used illicit drugs. The pelvic examination reveals a small, firm, non-tender, immobile cystic mass at the edge of the exocervical os with a yellow hue. The patient is otherwise asymptomatic and has no other complaints. The remainder of the physical examination is unremarkable. Which of the following is the next best step in management?A)No further intervention B)Colposcopy and biopsy C)Administration of oral contraceptives D)Administration of antibiotics\\n\\nA:'}\n",
            "{'question': 'Coves are given in:', 'correct_answer': 'Axial wall', 'options': ['A)Mesiofacial in gingival surface', 'B)Pulpal floor', 'C)Axial wall'], 'prompt': '\\n\\nQ:Coves are given in:A)Mesiofacial in gingival surface B)Pulpal floor C)Axial wall\\n\\nA:'}\n",
            "{'question': 'He constructed the following items located within the suboccipital triangle?', 'correct_answer': 'Occipital nerve', 'options': ['A)Internal carotid artery', 'B)Occipital artery', 'C)Occipital nerve'], 'prompt': '\\n\\nQ:He constructed the following items located within the suboccipital triangle?A)Internal carotid artery B)Occipital artery C)Occipital nerve\\n\\nA:'}\n",
            "{'question': 'Which of the following finding are there in iron deficiency anemia?', 'correct_answer': '| TIBC, | Ferritin, | Transferrin saturation', 'options': ['A)| TIBC, | Ferritin, | Transferrin saturation'], 'prompt': '\\n\\nQ:Which of the following finding are there in iron deficiency anemia?A)| TIBC, | Ferritin, | Transferrin saturation\\n\\nA:'}\n",
            "{'question': 'Ochronosis is due to the accumulation of?', 'correct_answer': 'Homogentisic acid', 'options': ['A)Glyoxylate', 'B)Xanthurenate', 'C)Homogentisic acid'], 'prompt': '\\n\\nQ:Ochronosis is due to the accumulation of?A)Glyoxylate B)Xanthurenate C)Homogentisic acid\\n\\nA:'}\n",
            "{'question': 'Coronary blood flow is maximum during which phase of cardiac cycle:', 'correct_answer': 'Isovolumic relaxation phase', 'options': ['A)Ejection phase', 'B)Isovolumic contraction phase', 'C)Isovolumic relaxation phase'], 'prompt': '\\n\\nQ:Coronary blood flow is maximum during which phase of cardiac cycle:A)Ejection phase B)Isovolumic contraction phase C)Isovolumic relaxation phase\\n\\nA:'}\n",
            "{'question': 'Ankle brachial pressure index is', 'correct_answer': '1', 'options': ['A)10', 'B)1', 'C)None'], 'prompt': '\\n\\nQ:Ankle brachial pressure index isA)10 B)1 C)None\\n\\nA:'}\n",
            "{'question': 'Site of injection in gluteus ?', 'correct_answer': 'Superolateral', 'options': ['A)Superolateral', 'B)Superomedial', 'C)Inferomedial'], 'prompt': '\\n\\nQ:Site of injection in gluteus ?A)Superolateral B)Superomedial C)Inferomedial\\n\\nA:'}\n",
            "{'question': 'Cotton wool skull is a radiological feature of -', 'correct_answer': 'Pagets disease', 'options': ['A)Eosinophilic granuloma', 'B)Pagets disease', 'C)Fibrous dysplasia'], 'prompt': '\\n\\nQ:Cotton wool skull is a radiological feature of -A)Eosinophilic granuloma B)Pagets disease C)Fibrous dysplasia\\n\\nA:'}\n",
            "{'question': '‘La facies sympathiteque’ is indicative of–', 'correct_answer': 'Ante-mortemhanging', 'options': ['A)Throttling', 'B)Post-mortemhanging', 'C)Ante-mortemhanging'], 'prompt': '\\n\\nQ:‘La facies sympathiteque’ is indicative of–A)Throttling B)Post-mortemhanging C)Ante-mortemhanging\\n\\nA:'}\n",
            "{'question': 'Optic tract lesion causes ?', 'correct_answer': \"Wernicke's hemianopic pupil\", 'options': ['A)Amauratic pupil', \"B)Wernicke's hemianopic pupil\", 'C)None'], 'prompt': \"\\n\\nQ:Optic tract lesion causes ?A)Amauratic pupil B)Wernicke's hemianopic pupil C)None\\n\\nA:\"}\n",
            "{'question': 'Multiple canals in mandibular premolars are seen in?', 'correct_answer': 'Africas', 'options': ['A)Caucians', 'B)Not Recalled', 'C)Africas'], 'prompt': '\\n\\nQ:Multiple canals in mandibular premolars are seen in?A)Caucians B)Not Recalled C)Africas\\n\\nA:'}\n",
            "{'question': 'RBC should be transfused:', 'correct_answer': 'With a 20-22 G needle within 4 hours of issue from the blood bank', 'options': [\"A)With a 20-22 G needle within 4 hours of receiving at the patient's side\", 'B)With a 20-22 G needle within 4 hours of issue from the blood bank', \"C)With a 18-20 G needle within 4 hours of receiving at the patient's side\"], 'prompt': \"\\n\\nQ:RBC should be transfused:A)With a 20-22 G needle within 4 hours of receiving at the patient's side B)With a 20-22 G needle within 4 hours of issue from the blood bank C)With a 18-20 G needle within 4 hours of receiving at the patient's side\\n\\nA:\"}\n",
            "{'question': \"Hutchinson's triad includes?\", 'correct_answer': 'Interstitial keratitis, malformed molars, sensorineural deafness', 'options': ['A)Interstitial keratitis, malformed molars, sensorineural deafness', 'B)Interstitial keratitis, malformed molars, conductive deafness', 'C)Conductive deafness, malformed incisors, interstitial keratitis'], 'prompt': \"\\n\\nQ:Hutchinson's triad includes?A)Interstitial keratitis, malformed molars, sensorineural deafness B)Interstitial keratitis, malformed molars, conductive deafness C)Conductive deafness, malformed incisors, interstitial keratitis\\n\\nA:\"}\n",
            "{'question': 'DOC for medical management of ectopic pegnancy ?', 'correct_answer': 'Intramuscular Methotrexate', 'options': ['A)Actinomycin D', 'B)Intramuscular Methotrexate', 'C)PGf2 alpha'], 'prompt': '\\n\\nQ:DOC for medical management of ectopic pegnancy ?A)Actinomycin D B)Intramuscular Methotrexate C)PGf2 alpha\\n\\nA:'}\n",
            "{'question': 'Which of the following presents with hypokalemia and metabolic acidosis ?', 'correct_answer': 'Diarrhea', 'options': ['A)Nasogastic suction', 'B)Diarrhea', 'C)Vomiting'], 'prompt': '\\n\\nQ:Which of the following presents with hypokalemia and metabolic acidosis ?A)Nasogastic suction B)Diarrhea C)Vomiting\\n\\nA:'}\n",
            "{'question': 'Optimum BMP dose for osteogenesis is:', 'correct_answer': '50 μg/mL', 'options': ['A)2.5 μg/mL', 'B)50 μg/mL', 'C)100 μg/mL'], 'prompt': '\\n\\nQ:Optimum BMP dose for osteogenesis is:A)2.5 μg/mL B)50 μg/mL C)100 μg/mL\\n\\nA:'}\n",
            "{'question': 'Enzyme that catalyses the transfer of a molecule group from one molecule to another is:', 'correct_answer': 'Transferases', 'options': ['A)Oxidases', 'B)Lipases', 'C)Transferases'], 'prompt': '\\n\\nQ:Enzyme that catalyses the transfer of a molecule group from one molecule to another is:A)Oxidases B)Lipases C)Transferases\\n\\nA:'}\n",
            "{'question': 'Most common presenting feature of complete mole is ?', 'correct_answer': 'Bleeding per vaginum', 'options': ['A)Bleeding per vaginum', 'B)Amenorrhoea', 'C)Vomiting'], 'prompt': '\\n\\nQ:Most common presenting feature of complete mole is ?A)Bleeding per vaginum B)Amenorrhoea C)Vomiting\\n\\nA:'}\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(new_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8719trqQuuP4",
        "outputId": "5632fa88-0983-4486-e67b-d6319c1d1e1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "16921"
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_data[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZHb0NiVEvY8Z",
        "outputId": "5760fedc-3fc9-42d4-8ca7-ff3a10eed805"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'Chordoma arises from:',\n",
              " 'correct_answer': 'Notochord',\n",
              " 'options': ['A)Notochord',\n",
              "  \"B)Luschka's bursa\",\n",
              "  'C)Pharyngeal bursa',\n",
              "  \"D)Rathke's pouch\"],\n",
              " 'prompt': \"\\n\\nQ:Chordoma arises from:A)Notochord B)Luschka's bursa C)Pharyngeal bursa D)Rathke's pouch\\n\\nA:\",\n",
              " 'output1': '\\n<bot>: Chordoma arises from the remnants of the notochord, a structure that is present in the embryo and then disappears.\\n\\n',\n",
              " 'output2': \"\\n<bot>: Because chordoma arises from the remnants of Luschka's bursa, which is a type of membranous structure that forms in the pharynx during embryonic development.\",\n",
              " 'output3': '\\n<bot>: Because chordoma arises from the pharyngeal bursa, which is a remnant of the pharyngeal arch.\\n<human>',\n",
              " 'output4': \"\\n<bot>: Because chordoma arises from a type of cell called a chordoma cell, which is found in Rathke's pouch. Rathke's pouch is a structure that is present in the neck and is thought to be the site of origin of chordoma.\\n\\n\",\n",
              " 'max_len': 'output4',\n",
              " 'correct_idx': 1}"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"corrected_medhalt_max_len_bad_op.json\", \"w\") as file:\n",
        "  json.dump(new_data, file)"
      ],
      "metadata": {
        "id": "iwsjh-upvtJg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### standardizing generations"
      ],
      "metadata": {
        "id": "oSBQ4Rr-wsC_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# standardize correct op and max_len op\n",
        "pbar = tqdm.tqdm(new_data)\n",
        "std_data = []\n",
        "for i in pbar:\n",
        "  prompt = \"\\n\\n####user\\n: \" + i[\"question\"] + \" \".join(i[\"options\"])\n",
        "  cor_gen = \"output\" + str(i[\"correct_idx\"])\n",
        "  cor_gen = i[cor_gen]\n",
        "  cor_gen = cor_gen.replace(\"<human>\", \"\").replace(\"\\n\", \"\").replace(\"\\n<bot>:\", \"\\n####assistant\\n\").replace(\"<bot>\", \"\\n####assistant\\n\")\n",
        "\n",
        "  op2 = i[\"max_len\"]\n",
        "  op2_gen = i[op2]\n",
        "  op2_gen = op2_gen.replace(\"<human>\", \"\").replace(\"\\n\", \"\").replace(\"\\n<bot>:\", \"\\n####assistant\\n\").replace(\"<bot>\", \"\\n####assistant\\n\")\n",
        "\n",
        "  if \"\\n####assistant\\n\" in cor_gen:\n",
        "    i[\"output\" + str(i[\"correct_idx\"])] = cor_gen\n",
        "    i[op2] = op2_gen\n",
        "    std_data.append(i)\n",
        "  else:\n",
        "    continue"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VQ_mHLPfwM-F",
        "outputId": "e0ad8bee-798a-4f5d-95f8-583aecc548c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 16921/16921 [00:00<00:00, 82592.50it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"zero_shot_medhalt.json\", \"w\") as file:\n",
        "  json.dump(std_data, file)"
      ],
      "metadata": {
        "id": "YnQutxcH3gIe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# basic checks what is the len of the correct option and the ideal bad option\n",
        "# also all A) B) C) D) should not be mentioned in\n",
        "reject_list = []\n",
        "approve_list = []\n",
        "pbar = tqdm.tqdm(std_data)\n",
        "for i in pbar:\n",
        "  rej_op = i[\"output\" + str(i[\"correct_idx\"])]\n",
        "  sel_op = i[i[\"max_len\"]]\n",
        "\n",
        "  if int(\"A)\" in rej_op) + int(\"B)\" in rej_op) + int(\"C)\" in rej_op) + int(\"D)\" in rej_op)>1:\n",
        "    reject_list.append(i)\n",
        "  elif int(\"A)\" in sel_op) + int(\"B)\" in sel_op) + int(\"C)\" in sel_op) + int(\"D)\" in sel_op)>1:\n",
        "    reject_list.append(i)\n",
        "\n",
        "  elif len(rej_op)<10 or len(sel_op)<10:\n",
        "    reject_list.append(i)\n",
        "  else:\n",
        "    approve_list.append(i)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k4SQMQqS0iuw",
        "outputId": "e298ef45-5901-4940-b4f2-23f7ef1329d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 15699/15699 [00:00<00:00, 110336.66it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(reject_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z9T9OAju4_0o",
        "outputId": "3107786c-c9b2-4676-ff35-3537d29e5bbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "579"
            ]
          },
          "metadata": {},
          "execution_count": 158
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(approve_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "auCyz9s45SS7",
        "outputId": "fda528fa-9816-4992-f3ff-bcf4e6a79335"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15120"
            ]
          },
          "metadata": {},
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"zero_shot_medhalt_cleaned.json\", \"w\") as file:\n",
        "  json.dump(approve_list, file)"
      ],
      "metadata": {
        "id": "dGAevai963E8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GtcjLtif8G-z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "L5QZShTL-QDV"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3331fb4366404d58a16e7d20c0bd9097": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_43a41c9e370a4c91b449a07b18e6f1a1",
              "IPY_MODEL_5db1eecb53e04f59ab90284612472c78",
              "IPY_MODEL_3277b6349a0040d986d6b7a39e9139b6"
            ],
            "layout": "IPY_MODEL_f6723587d1ae4ccd96d12a85294e9e0c"
          }
        },
        "43a41c9e370a4c91b449a07b18e6f1a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f04b13c4dfe345a59f0a17f509c82bcf",
            "placeholder": "​",
            "style": "IPY_MODEL_62ea33cfc2c94bc284528b08d4b8f3c5",
            "value": "Downloading readme: 100%"
          }
        },
        "5db1eecb53e04f59ab90284612472c78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ed59712d55146d599c0e71afe142edc",
            "max": 4797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_641fda76a3a445ea89612cb18122c933",
            "value": 4797
          }
        },
        "3277b6349a0040d986d6b7a39e9139b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a644ddc949fd43b4a6f06a831a417a9c",
            "placeholder": "​",
            "style": "IPY_MODEL_5243c34d06c94c6cbfc7a969dd1ef052",
            "value": " 4.80k/4.80k [00:00&lt;00:00, 106kB/s]"
          }
        },
        "f6723587d1ae4ccd96d12a85294e9e0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f04b13c4dfe345a59f0a17f509c82bcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62ea33cfc2c94bc284528b08d4b8f3c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ed59712d55146d599c0e71afe142edc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "641fda76a3a445ea89612cb18122c933": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a644ddc949fd43b4a6f06a831a417a9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5243c34d06c94c6cbfc7a969dd1ef052": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cd41ff734ca148f59afaa74fd11fc3c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_59467a6cbb024d0da5673f24d56672f4",
              "IPY_MODEL_cb66852d6fe145fb847d2ae8c451e924",
              "IPY_MODEL_087b4b5119e94f58b41d80fad5d806e3"
            ],
            "layout": "IPY_MODEL_b507a0731fce4a4284cdb119bfcabb44"
          }
        },
        "59467a6cbb024d0da5673f24d56672f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45f6e8fac26c41b7b2decf0efe18890a",
            "placeholder": "​",
            "style": "IPY_MODEL_be47fc28fa454b948bc73e78f7359bc5",
            "value": "Downloading data files: 100%"
          }
        },
        "cb66852d6fe145fb847d2ae8c451e924": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f15c408406f4dc2acc12f6b0b550bb1",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_13cc6346fc3a4b81bf5f3e4b5ca826dd",
            "value": 1
          }
        },
        "087b4b5119e94f58b41d80fad5d806e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64a103b4c5d043779a5312f48cdbf591",
            "placeholder": "​",
            "style": "IPY_MODEL_90a5a88b07a041ecbde0dbb3813c4b8e",
            "value": " 1/1 [00:01&lt;00:00,  1.04s/it]"
          }
        },
        "b507a0731fce4a4284cdb119bfcabb44": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45f6e8fac26c41b7b2decf0efe18890a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be47fc28fa454b948bc73e78f7359bc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2f15c408406f4dc2acc12f6b0b550bb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13cc6346fc3a4b81bf5f3e4b5ca826dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "64a103b4c5d043779a5312f48cdbf591": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90a5a88b07a041ecbde0dbb3813c4b8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "43ef7e4cacbb420a9ef05feacd13a27e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2e806f25f6354388b5bc462c369cb8ae",
              "IPY_MODEL_080b9a400c884b01b3a58c7c81520413",
              "IPY_MODEL_07e67c2ea89c4b73b58ffed70b1e49ed"
            ],
            "layout": "IPY_MODEL_febc5b7b0a6a49259c9eb0562def85d9"
          }
        },
        "2e806f25f6354388b5bc462c369cb8ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9c7c55cf92e4e0ca682799652e5a9b9",
            "placeholder": "​",
            "style": "IPY_MODEL_5a1fbffb723f4e89a7638d280dc21f9a",
            "value": "Downloading data: 100%"
          }
        },
        "080b9a400c884b01b3a58c7c81520413": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69433c9cdb204888ab681809c8c63367",
            "max": 9925083,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f83881a501a24de69646f84c2220723c",
            "value": 9925083
          }
        },
        "07e67c2ea89c4b73b58ffed70b1e49ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_add36d6197904e03991eb0e05194e7e3",
            "placeholder": "​",
            "style": "IPY_MODEL_4e887d54efe24cde8a473b8579676acd",
            "value": " 9.93M/9.93M [00:01&lt;00:00, 10.6MB/s]"
          }
        },
        "febc5b7b0a6a49259c9eb0562def85d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9c7c55cf92e4e0ca682799652e5a9b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a1fbffb723f4e89a7638d280dc21f9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "69433c9cdb204888ab681809c8c63367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f83881a501a24de69646f84c2220723c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "add36d6197904e03991eb0e05194e7e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e887d54efe24cde8a473b8579676acd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d7a359c7bbb47b5a81c8d95bd804c9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8616ccd8ab0b4fa699a02a95d2dc4c91",
              "IPY_MODEL_398491f9144c43dfa96757f70c919c87",
              "IPY_MODEL_82fde32070dd4953bc3e5a40c741ef70"
            ],
            "layout": "IPY_MODEL_1ea65dc5f39049478c835b4ce20d6b11"
          }
        },
        "8616ccd8ab0b4fa699a02a95d2dc4c91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed99cdc6ed3e42ef888715e5baa66546",
            "placeholder": "​",
            "style": "IPY_MODEL_a5ac8bb88c78444a902c559a6f698413",
            "value": "Extracting data files: 100%"
          }
        },
        "398491f9144c43dfa96757f70c919c87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ed49878275d4d6994bbe926933a3243",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3e5db9789f1646768b403b8da6012c3b",
            "value": 1
          }
        },
        "82fde32070dd4953bc3e5a40c741ef70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb66a8f7b45c41e1afa00f111c41d666",
            "placeholder": "​",
            "style": "IPY_MODEL_7829b2f1bd5d42d78e38269a69e488af",
            "value": " 1/1 [00:00&lt;00:00, 38.79it/s]"
          }
        },
        "1ea65dc5f39049478c835b4ce20d6b11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed99cdc6ed3e42ef888715e5baa66546": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5ac8bb88c78444a902c559a6f698413": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7ed49878275d4d6994bbe926933a3243": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e5db9789f1646768b403b8da6012c3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb66a8f7b45c41e1afa00f111c41d666": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7829b2f1bd5d42d78e38269a69e488af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8b76431d3a034b80a27950c1ce3637d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e29911f5da30487a95de79566f0b9e2d",
              "IPY_MODEL_a68a1df5bc1b4097a5527cbb17923da1",
              "IPY_MODEL_dae58c0ac00a416b91dba7ae6f17321f"
            ],
            "layout": "IPY_MODEL_48c8b41d5017457cb1014e432dd76bd7"
          }
        },
        "e29911f5da30487a95de79566f0b9e2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae79e2b26ecf4cf49a8a0a43011858ae",
            "placeholder": "​",
            "style": "IPY_MODEL_8859e66caf7942809d5b7a122791162d",
            "value": "Generating train split: "
          }
        },
        "a68a1df5bc1b4097a5527cbb17923da1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a4f1c8772d84e01b07ba4d7bc57772e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ef36cd3a5bc243a0bc8b37910f7706ea",
            "value": 1
          }
        },
        "dae58c0ac00a416b91dba7ae6f17321f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d9b2599c8754926a359f934b6533448",
            "placeholder": "​",
            "style": "IPY_MODEL_7cf927e0e5184e7bb4edac7a08155731",
            "value": " 18866/0 [00:00&lt;00:00, 29970.98 examples/s]"
          }
        },
        "48c8b41d5017457cb1014e432dd76bd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae79e2b26ecf4cf49a8a0a43011858ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8859e66caf7942809d5b7a122791162d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a4f1c8772d84e01b07ba4d7bc57772e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "ef36cd3a5bc243a0bc8b37910f7706ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7d9b2599c8754926a359f934b6533448": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7cf927e0e5184e7bb4edac7a08155731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c8bf0286f5a9464197bf68fca2dd815d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_98360b4b8e9c4ce682a777efa4e1acea",
              "IPY_MODEL_4f81e2574b304e55afb539f461d9b887",
              "IPY_MODEL_4ec1faac11e545db81995eeb5a125bdb"
            ],
            "layout": "IPY_MODEL_4fa845f08caf4d3289d2824fb38cdaa1"
          }
        },
        "98360b4b8e9c4ce682a777efa4e1acea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95b52c7f4636439aa99da75040469938",
            "placeholder": "​",
            "style": "IPY_MODEL_0ce2fc330b0b4a4b95e86c235dc30d7d",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "4f81e2574b304e55afb539f461d9b887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ae114d4ac874a058367dcfb31efd326",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2f62a6ac3f7c4775b6b6c02db3510740",
            "value": 28
          }
        },
        "4ec1faac11e545db81995eeb5a125bdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_41379302c0d64aceb6fc3a10f2b30d14",
            "placeholder": "​",
            "style": "IPY_MODEL_2c307ad586454159bc2abdcdc938c404",
            "value": " 28.0/28.0 [00:00&lt;00:00, 921B/s]"
          }
        },
        "4fa845f08caf4d3289d2824fb38cdaa1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95b52c7f4636439aa99da75040469938": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ce2fc330b0b4a4b95e86c235dc30d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ae114d4ac874a058367dcfb31efd326": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f62a6ac3f7c4775b6b6c02db3510740": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41379302c0d64aceb6fc3a10f2b30d14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c307ad586454159bc2abdcdc938c404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1894d4a6c92148faab148ed25ac1354e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_850fd1051fc34b8ba9aeb2ad0c1f18a8",
              "IPY_MODEL_534bcf3be79c453fb788643447216359",
              "IPY_MODEL_5bea594d2fc341069c24c74859c57983"
            ],
            "layout": "IPY_MODEL_1be958cc80614d988701fca0862a94f8"
          }
        },
        "850fd1051fc34b8ba9aeb2ad0c1f18a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b9e5569ebfe4b1db1073148f67fb612",
            "placeholder": "​",
            "style": "IPY_MODEL_43004ea6e0024b8b95ce3b5df797dd4a",
            "value": "vocab.txt: 100%"
          }
        },
        "534bcf3be79c453fb788643447216359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3efbb8257f24ce4a4ee943c1812920e",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f87f7b4a212a44a4881c9481edefe391",
            "value": 231508
          }
        },
        "5bea594d2fc341069c24c74859c57983": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2878d0e4925140e7b1f252e1915a8cab",
            "placeholder": "​",
            "style": "IPY_MODEL_c8e42577e8064ad8b113c7f65d170ab0",
            "value": " 232k/232k [00:00&lt;00:00, 1.62MB/s]"
          }
        },
        "1be958cc80614d988701fca0862a94f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b9e5569ebfe4b1db1073148f67fb612": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43004ea6e0024b8b95ce3b5df797dd4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f3efbb8257f24ce4a4ee943c1812920e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f87f7b4a212a44a4881c9481edefe391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2878d0e4925140e7b1f252e1915a8cab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8e42577e8064ad8b113c7f65d170ab0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7b48148aea0641c1bc8cea4b142c1122": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5654a5fb927649b6ba54ca11aa2c1827",
              "IPY_MODEL_111983a87f344d47aa99ae52d1bd2d62",
              "IPY_MODEL_cc4cf3e1cd9543479991f4fac937a556"
            ],
            "layout": "IPY_MODEL_2806046261944b10bd0bae0b5201ccea"
          }
        },
        "5654a5fb927649b6ba54ca11aa2c1827": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea13a40ee8b5490cac9211203b46b53b",
            "placeholder": "​",
            "style": "IPY_MODEL_876cdc9a17e44d84a54e0683403bc5b8",
            "value": "tokenizer.json: 100%"
          }
        },
        "111983a87f344d47aa99ae52d1bd2d62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd7b54168c2e474892ef67dd6f901358",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_509ab65d1a1541eb8b8355a1c0ab56bc",
            "value": 466062
          }
        },
        "cc4cf3e1cd9543479991f4fac937a556": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b46135e185624e46a99ad19fdc0aeecb",
            "placeholder": "​",
            "style": "IPY_MODEL_26b7958379ec4283bfb71d72d4bdf4a5",
            "value": " 466k/466k [00:00&lt;00:00, 4.68MB/s]"
          }
        },
        "2806046261944b10bd0bae0b5201ccea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea13a40ee8b5490cac9211203b46b53b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "876cdc9a17e44d84a54e0683403bc5b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fd7b54168c2e474892ef67dd6f901358": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "509ab65d1a1541eb8b8355a1c0ab56bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b46135e185624e46a99ad19fdc0aeecb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26b7958379ec4283bfb71d72d4bdf4a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "169310639642494d98638f7aaf09234c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ed7855f486a140f18d6507d0333571cf",
              "IPY_MODEL_b6beb8e5efec4b57972a02560909428d",
              "IPY_MODEL_e95457532e8b43cd958f3d1d85c4679a"
            ],
            "layout": "IPY_MODEL_55a15c8ff61b4c8c82a530fc60215e94"
          }
        },
        "ed7855f486a140f18d6507d0333571cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd78b614846e4d30b855b34d40b8264d",
            "placeholder": "​",
            "style": "IPY_MODEL_0ea5f422e67e452cbc55565cd675760c",
            "value": "config.json: 100%"
          }
        },
        "b6beb8e5efec4b57972a02560909428d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_533c98b95af54c8aa58c84101099264e",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3702b1ebd6114a45b5d0c4f7946643e7",
            "value": 570
          }
        },
        "e95457532e8b43cd958f3d1d85c4679a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f65fd8aba7374e5aa22408f6d4b1aebc",
            "placeholder": "​",
            "style": "IPY_MODEL_c695375257a14f08a3ab00534b195954",
            "value": " 570/570 [00:00&lt;00:00, 11.5kB/s]"
          }
        },
        "55a15c8ff61b4c8c82a530fc60215e94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd78b614846e4d30b855b34d40b8264d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ea5f422e67e452cbc55565cd675760c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "533c98b95af54c8aa58c84101099264e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3702b1ebd6114a45b5d0c4f7946643e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f65fd8aba7374e5aa22408f6d4b1aebc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c695375257a14f08a3ab00534b195954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b6992916e4d141c99980f512df1f3782": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dd954c7e5a00426b911dd8be94070456",
              "IPY_MODEL_dbe7935becd24d2a96dc1d599a402ad0",
              "IPY_MODEL_202a2a217f554b57a685d1387239a2af"
            ],
            "layout": "IPY_MODEL_b6b1dbede9144c5a88933897b1754c1d"
          }
        },
        "dd954c7e5a00426b911dd8be94070456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eac94ad5275f4fc6aa5de9bc3eccc0b5",
            "placeholder": "​",
            "style": "IPY_MODEL_95b4fc89ff4341399ef08c08b2b1a8d6",
            "value": "model.safetensors: 100%"
          }
        },
        "dbe7935becd24d2a96dc1d599a402ad0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ae9ca72a1694e638be56a62c17408f6",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_60e717050fd34ea09a51315a841b80de",
            "value": 440449768
          }
        },
        "202a2a217f554b57a685d1387239a2af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c32d7cc872b34b68ac6a28453578ed38",
            "placeholder": "​",
            "style": "IPY_MODEL_7aa0e089eb224e5790bfad0a8cbcfa8c",
            "value": " 440M/440M [00:04&lt;00:00, 98.3MB/s]"
          }
        },
        "b6b1dbede9144c5a88933897b1754c1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eac94ad5275f4fc6aa5de9bc3eccc0b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95b4fc89ff4341399ef08c08b2b1a8d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5ae9ca72a1694e638be56a62c17408f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60e717050fd34ea09a51315a841b80de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c32d7cc872b34b68ac6a28453578ed38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7aa0e089eb224e5790bfad0a8cbcfa8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}