benchmarking_hallucination_model/LLM_as_judge_and_TLM.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gdq2ZGSlY1wH"
      },
      "source": [
        "# LLM-as-judge and TLM on ELI5 dataset\n",
        "\n",
        "## Setup\n",
        "\n",
        "The two main requirements to replicate are an OPENAI_API_KEY and a TLM_API_KEY.\n",
        "\n",
        "You can get an OPENAI_API_KEY by signing up for OpenAI at https://platform.openai.com/signup.\n",
        "\n",
        "You can try TLM for free at https://cleanlab.ai/tlm/\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VQqMpRVP3J9P"
      },
      "outputs": [],
      "source": [
        "%pip install datasets cleanlab_studio openai --quiet"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "metadata": {
        "id": "AcrGn4gLrDHj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1EiWmdBI3J9S"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "REQUIRED_CREDS = [\n",
        "    \"OPENAI_API_KEY\",  # https://platform.openai.com/\n",
        "    \"TLM_API_KEY\",  # https://app.cleanlab.ai/tlm\n",
        "    ]\n",
        "\n",
        "\n",
        "# Try/except in case you're using google colab for creds\n",
        "try:\n",
        "  from google.colab import userdata\n",
        "  for cred in REQUIRED_CREDS:\n",
        "    os.environ[cred] = userdata.get(cred)\n",
        "except ImportError:\n",
        "  pass\n",
        "\n",
        "# Also support dotenv, if preferred\n",
        "try:\n",
        "  from dotenv import load_dotenv\n",
        "  load_dotenv()\n",
        "except ImportError:\n",
        "  pass\n",
        "\n",
        "for cred in REQUIRED_CREDS:\n",
        "    assert cred in os.environ, f\"{cred} not found in environment variables. Please set it before proceeding.\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0lsEwybR3J9S",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279,
          "referenced_widgets": [
            "e5574ff5b6664d6facd5646304b348b4",
            "146d1103a9574e06afcb862f9e387e17",
            "b696f62ebc70468bbb604ec313cf033c",
            "aa9ab7df567748de9f3957b869652d1a",
            "1e54fd40b722404abbd08417e051de89",
            "bf43708ff4034462a38e56f7067a6bdf",
            "b847f1fae0eb4ee1835a2bfa5fd247e4",
            "b683cd9bee184abea24844fb2240bf3e",
            "e7855f20fdfe4062a58428ba6f24b5b6",
            "0500a88439874ca1a748b0c8841b2ea1",
            "a3e608e8adc24da58ff98abe1fa4c098",
            "6d2936de62fa4ba984965dc070703925",
            "b33a5a06f104419f814537aa5980bc5e",
            "f6621f23985944a1acb9daa25b1e77b0",
            "ff57b72d909a4f61adbf56caa30b2879",
            "42dae1cdd19c4ea7aabb735fada6544c",
            "2266539dec0546d98e21e0fa81571ebf",
            "f17f58b13d1e4167b4a0db6f284bc221",
            "5900264ee32944d4bb2a31a7ad50cebd",
            "5297f6371f744f4ebb082877d787e1de",
            "7854c0d933ad4b49b21453e6697334da",
            "691f24fc57384a498c1d48f1796b1b94",
            "0e76b7dad697481daf0f6fd0f74ca29b",
            "8c9f9480951c43baa7830ccc6089c09c",
            "f620605aaadf45e89c585452047d1c13",
            "48058b8aa4c240ae9c5645db5c65f8ee",
            "5d72b7b4707e404b96017510bb1328f0",
            "fc9838f7d23b458281aa790d42b77c81",
            "86e612bb868c4accaf07543bffa58d0d",
            "1ec551ed9de94753b9a17e794938eab2",
            "00c73d99ab8c46c096431a78f292c7a5",
            "72567652d7a1471294ff05c564e93ad7",
            "12866bd620574403a8c8b94fdf538b67",
            "1c8cf500c16f4a3a8345b6fc61184dbd",
            "6e60c60a05394708abc01127a65c600b",
            "9f72ed068c43438cb85484f34f4866f8",
            "eb9d6ffe1e8d41cfbac7865cd20b9830",
            "881d913ecb4d44b29f49b3e125152dc9",
            "32d5642ec1a0493c8e8bf9e1870387ee",
            "042314e7fa404c54a47012568a3a7067",
            "2a2552ac6cbc4bffacc42870b646b21f",
            "2ce310d4137d456d920d388882b09719",
            "84961f5009414d3fb37fe8871c98f45c",
            "e6e4928ee2c04803b6830e92736f2223",
            "d81af0dcaeeb4cd6a8937fbad0ae02f1",
            "7e7ea579720440638aaf14a48d224a66",
            "1fc781ff24e74224b5546a2c897b7220",
            "1791fc4f7ddb480abb79d012486fe6ca",
            "07ab60f0178f465ab8ec7b3dfe7fd384",
            "dc6c9f6f66de4ba389975306afdd1f25",
            "10221957f03347af99b6c1d5c0694f6b",
            "62966b2d246242fea72ad72b85485c16",
            "9f2535a0af744b1db11cd0580064543a",
            "cf960782c33a4f2bbe646844fe1d6880",
            "88e29be850ea431aaab4fe21d09400a6"
          ]
        },
        "outputId": "64e1bb6a-371e-416a-9428-2c41163a84b2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "README.md:   0%|          | 0.00/480 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e5574ff5b6664d6facd5646304b348b4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "test-00000-of-00001.parquet:   0%|          | 0.00/48.6k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6d2936de62fa4ba984965dc070703925"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "train-00000-of-00001.parquet:   0%|          | 0.00/74.9k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e76b7dad697481daf0f6fd0f74ca29b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/56 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1c8cf500c16f4a3a8345b6fc61184dbd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/100 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d81af0dcaeeb4cd6a8937fbad0ae02f1"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "ds = load_dataset(\"explodinggradients/ELI5\")\n",
        "data = ds[\"train\"].to_pandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "uZibFtvP3J9S",
        "outputId": "cd42357b-76b7-470c-d5e8-a58056fc1cd1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                          user_input  \\\n",
              "0  What is the significance of evolutionary devel...   \n",
              "1  What is the Theory of Sediment Transport and h...   \n",
              "2  What is the Theory of Isostasy and how does it...   \n",
              "3  What are the key concepts in the Theory of Dig...   \n",
              "4  What is the Germ Theory of Disease and how did...   \n",
              "\n",
              "                                           reference  \\\n",
              "0  Evolutionary developmental biology, often refe...   \n",
              "1  The Theory of Sediment Transport is a fundamen...   \n",
              "2  The Theory of Isostasy is a concept in geology...   \n",
              "3  The Theory of Digital Computation encompasses ...   \n",
              "4  The Germ Theory of Disease is a scientific the...   \n",
              "\n",
              "                                            response  target  \n",
              "0  Evolutionary developmental biology, or 'evo-de...       1  \n",
              "1  The Theory of Sediment Transport is like a big...       0  \n",
              "2  The Theory of Isostasy is like saying the Eart...       0  \n",
              "3  The Theory of Digital Computation is like a bi...       1  \n",
              "4  The Germ Theory of Disease is like saying tiny...       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d9d26957-fd94-45ee-bdd9-bec007a6e1e1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_input</th>\n",
              "      <th>reference</th>\n",
              "      <th>response</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What is the significance of evolutionary devel...</td>\n",
              "      <td>Evolutionary developmental biology, often refe...</td>\n",
              "      <td>Evolutionary developmental biology, or 'evo-de...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the Theory of Sediment Transport and h...</td>\n",
              "      <td>The Theory of Sediment Transport is a fundamen...</td>\n",
              "      <td>The Theory of Sediment Transport is like a big...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What is the Theory of Isostasy and how does it...</td>\n",
              "      <td>The Theory of Isostasy is a concept in geology...</td>\n",
              "      <td>The Theory of Isostasy is like saying the Eart...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What are the key concepts in the Theory of Dig...</td>\n",
              "      <td>The Theory of Digital Computation encompasses ...</td>\n",
              "      <td>The Theory of Digital Computation is like a bi...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>What is the Germ Theory of Disease and how did...</td>\n",
              "      <td>The Germ Theory of Disease is a scientific the...</td>\n",
              "      <td>The Germ Theory of Disease is like saying tiny...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d9d26957-fd94-45ee-bdd9-bec007a6e1e1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d9d26957-fd94-45ee-bdd9-bec007a6e1e1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d9d26957-fd94-45ee-bdd9-bec007a6e1e1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-cba500ad-0529-4c36-b87d-ac8a64e16350\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cba500ad-0529-4c36-b87d-ac8a64e16350')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-cba500ad-0529-4c36-b87d-ac8a64e16350 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"user_input\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"What is the Theory of Signal Transduction and how does it explain cellular communication?\",\n          \"What is the Theory of Milankovitch Cycles and how does it explain climate change?\",\n          \"What is the Theory of Black Holes and how does it explain the nature of these cosmic phenomena?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"The Theory of Signal Transduction describes the process by which cells respond to external signals through a series of molecular events. This process begins when a signaling molecule, such as a hormone or neurotransmitter, binds to a receptor on the cell surface. This binding triggers a cascade of intracellular reactions, often involving the activation of proteins and second messengers, which ultimately lead to a specific cellular response. Signal transduction is crucial for various cellular functions, including growth, differentiation, and metabolism, and plays a key role in maintaining homeostasis within an organism.\",\n          \"The Theory of Milankovitch Cycles explains the impact of Earth's movements on its climate over thousands of years. It is based on three main types of astronomical cycles: eccentricity, axial tilt, and precession. Eccentricity refers to the shape of Earth's orbit around the sun, which changes from more circular to more elliptical over a period of about 100,000 years. Axial tilt, or obliquity, changes the angle of Earth's axis relative to its orbit, affecting the distribution of sunlight on Earth's surface over a 41,000-year cycle. Precession is the wobble in Earth's rotation, which alters the timing of the seasons over a 26,000-year cycle. These cycles influence the amount and distribution of solar energy received by Earth, contributing to long-term climate patterns such as ice ages and interglacial periods.\",\n          \"The Theory of Black Holes is a fundamental concept in astrophysics that describes regions in space where the gravitational pull is so strong that nothing, not even light, can escape. This theory is rooted in Einstein's General Theory of Relativity, which predicts that when a massive star collapses under its own gravity, it can form a singularity\\u2014a point of infinite density. Surrounding this singularity is the event horizon, the boundary beyond which nothing can return. Black holes are characterized by their mass, charge, and angular momentum, and they play a crucial role in understanding the dynamics of galaxies and the evolution of the universe.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"response\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"The Theory of Signal Transduction is like a game of telephone inside our cells. It starts when a special molecule, like a hormone, knocks on the cell's door. This knock sets off a chain reaction inside the cell, like a line of dominoes falling, which tells the cell what to do next. This process helps cells grow, change, and keep everything in balance, just like how our bodies stay healthy!\",\n          \"The Theory of Milankovitch Cycles is like a big clock that shows how Earth's movements change the weather over a long, long time. There are three main parts: First, eccentricity is about how Earth's path around the sun changes shape, like a circle turning into an oval, every 100,000 years. Second, axial tilt is how Earth leans a bit more or less, changing how sunlight hits it, every 41,000 years. Third, precession is like a wobble in Earth's spin, changing when seasons happen, every 26,000 years. These movements change how much sunlight Earth gets, which can make big weather changes like ice ages!\",\n          \"Black holes are like giant space vacuum cleaners that suck up everything, even light, so we can't see them! This idea comes from a smart guy named Einstein. He said that when a really big star gets squished down super small, it makes a black hole. Around the black hole is a line called the event horizon, and if anything crosses it, it can't come back. Black holes help us learn about how galaxies move and how the universe changes over time.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_8XwGILV3J9U"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from cleanlab_studio import Studio\n",
        "\n",
        "studio = Studio(os.environ[\"TLM_API_KEY\"])\n",
        "tlm = studio.TLM()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vm94-G4y3J9U"
      },
      "source": [
        "Since `tlm.get_trustworthy_score` takes `prompt` and `response`, we'll turn the `context` and `question` into a single prompt:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bHfUUFWV3J9U"
      },
      "outputs": [],
      "source": [
        "data[\"context_and_question_as_prompt\"] = data.apply(\n",
        "    lambda row: f\"Answer the QUESTION using information **only** from CONTEXT, but answer using only ELI5 language.\\n\\nCONTEXT:\\n{row['reference']}\\nQUESTION:\\n{row['user_input']}\",\n",
        "    axis=1\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "id": "BH4RIRWPp2jn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FZt42kFt3J9V"
      },
      "source": [
        "## LLM-as-judge\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4dpr2hmO3J9V"
      },
      "outputs": [],
      "source": [
        "CONFIDENCE_PROMPT = \"\"\"Evaluate how confident you are that the given Answer is an accurate response to the Question, given the Answer was supposed to be written in ELI5 language while remaining factually accurate.\n",
        "Please assign a Score using the following 5-point scale:\n",
        "1: You are not confident that the Answer addresses the Question at all, the Answer may be entirely off-topic or irrelevant to the Question.\n",
        "2: You have low confidence that the Answer addresses the Question, there are doubts and uncertainties about the accuracy of the Answer.\n",
        "3: You have moderate confidence that the Answer addresses the Question, the Answer seems reasonably accurate and on-topic, but with room for improvement.\n",
        "4: You have high confidence that the Answer addresses the Question, the Answer provides accurate information that addresses most of the Question.\n",
        "5: You are extremely confident that the Answer addresses the Question, the Answer is highly accurate, relevant, and effectively addresses the Question in its entirety.\n",
        "The output should strictly use the following template: Explanation: [provide a brief reasoning you used to derive the rating Score] and then write 'Score: <rating>' on the last line.\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eg6n-5zSqqxk"
      },
      "source": [
        "Through trial and error, we also found the following score extraction regexes to be useful:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S7FYFQLZqqxl"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "\n",
        "regex_list = [\n",
        "    re.compile(\n",
        "        r\".*(?:^|\\n)\\s*score:?\\s*\\(?(?P<answer>one|two|three|four|five|[12345])\\)?\",\n",
        "        flags=re.DOTALL | re.IGNORECASE\n",
        "    ),\n",
        "    re.compile(\n",
        "        r\".*\\(?(?P<answer>one|two|three|four|five|[12345])\\)?\",\n",
        "        flags=re.DOTALL | re.IGNORECASE\n",
        "    )\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from textwrap import dedent\n",
        "\n",
        "def construct_confidence_prompt_final(context, question, response):\n",
        "    template = f\"\"\"Context:\\n{context}\\nQuestion:\\n{question}\\nAnswer:\\n{response}\\n\\n{CONFIDENCE_PROMPT}\"\"\"\n",
        "    return dedent(template)\n",
        "\n",
        "def parse_llm_as_judge_score(text: str):\n",
        "    try:\n",
        "        score = int(text.split(\"Score:\")[-1].strip())\n",
        "        return (score - 1) / 4  # Normalize to 0-1 range\n",
        "    except:\n",
        "        return np.nan\n",
        "\n",
        "print(construct_confidence_prompt_final(data['reference'][0], data['user_input'][0], data['response'][0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GKKy6BURKo0o",
        "outputId": "85e109e6-aecd-4d31-cfb1-f5d26ecc18b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Context:\n",
            "Evolutionary developmental biology, often referred to as 'evo-devo,' is significant because it provides insights into how developmental processes influence evolutionary changes. By studying the genetic and developmental mechanisms that lead to the formation of different structures in organisms, evo-devo helps explain how complex traits evolve and diversify. It bridges the gap between microevolutionary processes, such as genetic mutations, and macroevolutionary patterns, like the emergence of new species. This field has revealed that small changes in developmental genes can lead to significant morphological differences, offering a deeper understanding of the evolutionary history and relationships among species.\n",
            "Question:\n",
            "What is the significance of evolutionary developmental biology in understanding the evolution of species?\n",
            "Answer:\n",
            "Evolutionary developmental biology, or 'evo-devo,' is like a detective story about how living things change over time. It helps us understand how tiny changes in the way creatures grow can lead to big differences, like new body parts or even new species. By looking at how genes and growth work together, evo-devo shows us how small tweaks can make big changes in the animal world, helping us learn more about how all the different animals we see today came to be.\n",
            "\n",
            "Evaluate how confident you are that the given Answer is an accurate response to the Question, given the Answer was supposed to be written in ELI5 language while remaining factually accurate.\n",
            "Please assign a Score using the following 5-point scale:\n",
            "1: You are not confident that the Answer addresses the Question at all, the Answer may be entirely off-topic or irrelevant to the Question.\n",
            "2: You have low confidence that the Answer addresses the Question, there are doubts and uncertainties about the accuracy of the Answer.\n",
            "3: You have moderate confidence that the Answer addresses the Question, the Answer seems reasonably accurate and on-topic, but with room for improvement.\n",
            "4: You have high confidence that the Answer addresses the Question, the Answer provides accurate information that addresses most of the Question.\n",
            "5: You are extremely confident that the Answer addresses the Question, the Answer is highly accurate, relevant, and effectively addresses the Question in its entirety.\n",
            "The output should strictly use the following template: Explanation: [provide a brief reasoning you used to derive the rating Score] and then write 'Score: <rating>' on the last line.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "244V7Wyxqqxl"
      },
      "source": [
        "Creating a function to run both of these benchmarks on the ELI5 dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9ltqxAGG3J9V"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "\n",
        "from datetime import datetime\n",
        "\n",
        "def score_tlm(data):\n",
        "    tlm_scores = tlm.get_trustworthiness_score(data[\"context_and_question_as_prompt\"].tolist(), data[\"response\"].tolist())\n",
        "    data['tlm_score'] = [score['trustworthiness_score'] for score in tlm_scores]\n",
        "    print(f\"Finished computing TLM Scores: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n",
        "    return data\n",
        "\n",
        "def score_llm_as_judge(data):\n",
        "    gpt4o_mini = studio.TLM(\"base\", options={\"model\": \"gpt-4o-mini\"})\n",
        "    data[\"llm_as_judge_raw\"] = gpt4o_mini.prompt([construct_confidence_prompt_final(row['reference'], row[\"user_input\"], row[\"response\"]) for _, row in data.iterrows()])\n",
        "    data[\"llm_as_judge_score\"] = data[\"llm_as_judge_raw\"].apply(lambda res: parse_llm_as_judge_score(res['response']))\n",
        "    print(f\"Finished computing LLM-as-judge Scores: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n",
        "    return data\n",
        "\n",
        "def score_dataset_with_metrics(in_data: pd.DataFrame):\n",
        "    # We copy the input data to avoid mutating the original dataframe\n",
        "    data = in_data.copy()\n",
        "    print(f\"Starting to Score Dataset: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n",
        "\n",
        "    data = score_tlm(data)\n",
        "    data = score_llm_as_judge(data)\n",
        "\n",
        "    return data\n",
        "\n",
        "def save_to_json(data_df, file_name, column_name, dataset_name):\n",
        "    if os.path.exists(filename):\n",
        "    with open(file_path, 'r') as file:\n",
        "        results = json.load(file)\n",
        "    else:\n",
        "        results = {}\n",
        "\n",
        "    # Update the results JSON with new scores\n",
        "    scores = data_df[column_name].tolist()\n",
        "    results[dataset_name] = [{'score': score} for score in scores]\n",
        "\n",
        "    with open(filename, 'w') as file:\n",
        "      json.dump(results, file, indent=4)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "result = score_dataset_with_metrics(data)\n",
        "\n",
        "# Save results to model specific JSON\n",
        "file_name = 'TLM.json'\n",
        "column_name = 'tlm_score'\n",
        "dataset_name = 'ELI5'\n",
        "save_to_json(result, file_name, column_name, dataset_name)\n",
        "\n",
        "file_name = 'LLM-as-judge.json'\n",
        "column_name = 'llm_as_judge_score'\n",
        "dataset_name = 'ELI5'\n",
        "save_to_json(result, file_name, column_name, dataset_name)\n",
        "\n",
        "result.to_csv(f\"tlm_eli5.csv\")"
      ],
      "metadata": {
        "id": "DZ_aVdbY7OtM"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.5"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e5574ff5b6664d6facd5646304b348b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_146d1103a9574e06afcb862f9e387e17",
              "IPY_MODEL_b696f62ebc70468bbb604ec313cf033c",
              "IPY_MODEL_aa9ab7df567748de9f3957b869652d1a"
            ],
            "layout": "IPY_MODEL_1e54fd40b722404abbd08417e051de89"
          }
        },
        "146d1103a9574e06afcb862f9e387e17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf43708ff4034462a38e56f7067a6bdf",
            "placeholder": "​",
            "style": "IPY_MODEL_b847f1fae0eb4ee1835a2bfa5fd247e4",
            "value": "README.md: 100%"
          }
        },
        "b696f62ebc70468bbb604ec313cf033c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b683cd9bee184abea24844fb2240bf3e",
            "max": 480,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e7855f20fdfe4062a58428ba6f24b5b6",
            "value": 480
          }
        },
        "aa9ab7df567748de9f3957b869652d1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0500a88439874ca1a748b0c8841b2ea1",
            "placeholder": "​",
            "style": "IPY_MODEL_a3e608e8adc24da58ff98abe1fa4c098",
            "value": " 480/480 [00:00&lt;00:00, 8.64kB/s]"
          }
        },
        "1e54fd40b722404abbd08417e051de89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf43708ff4034462a38e56f7067a6bdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b847f1fae0eb4ee1835a2bfa5fd247e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b683cd9bee184abea24844fb2240bf3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7855f20fdfe4062a58428ba6f24b5b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0500a88439874ca1a748b0c8841b2ea1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3e608e8adc24da58ff98abe1fa4c098": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d2936de62fa4ba984965dc070703925": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b33a5a06f104419f814537aa5980bc5e",
              "IPY_MODEL_f6621f23985944a1acb9daa25b1e77b0",
              "IPY_MODEL_ff57b72d909a4f61adbf56caa30b2879"
            ],
            "layout": "IPY_MODEL_42dae1cdd19c4ea7aabb735fada6544c"
          }
        },
        "b33a5a06f104419f814537aa5980bc5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2266539dec0546d98e21e0fa81571ebf",
            "placeholder": "​",
            "style": "IPY_MODEL_f17f58b13d1e4167b4a0db6f284bc221",
            "value": "test-00000-of-00001.parquet: 100%"
          }
        },
        "f6621f23985944a1acb9daa25b1e77b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5900264ee32944d4bb2a31a7ad50cebd",
            "max": 48597,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5297f6371f744f4ebb082877d787e1de",
            "value": 48597
          }
        },
        "ff57b72d909a4f61adbf56caa30b2879": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7854c0d933ad4b49b21453e6697334da",
            "placeholder": "​",
            "style": "IPY_MODEL_691f24fc57384a498c1d48f1796b1b94",
            "value": " 48.6k/48.6k [00:00&lt;00:00, 534kB/s]"
          }
        },
        "42dae1cdd19c4ea7aabb735fada6544c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2266539dec0546d98e21e0fa81571ebf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f17f58b13d1e4167b4a0db6f284bc221": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5900264ee32944d4bb2a31a7ad50cebd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5297f6371f744f4ebb082877d787e1de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7854c0d933ad4b49b21453e6697334da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "691f24fc57384a498c1d48f1796b1b94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e76b7dad697481daf0f6fd0f74ca29b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8c9f9480951c43baa7830ccc6089c09c",
              "IPY_MODEL_f620605aaadf45e89c585452047d1c13",
              "IPY_MODEL_48058b8aa4c240ae9c5645db5c65f8ee"
            ],
            "layout": "IPY_MODEL_5d72b7b4707e404b96017510bb1328f0"
          }
        },
        "8c9f9480951c43baa7830ccc6089c09c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fc9838f7d23b458281aa790d42b77c81",
            "placeholder": "​",
            "style": "IPY_MODEL_86e612bb868c4accaf07543bffa58d0d",
            "value": "train-00000-of-00001.parquet: 100%"
          }
        },
        "f620605aaadf45e89c585452047d1c13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ec551ed9de94753b9a17e794938eab2",
            "max": 74895,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_00c73d99ab8c46c096431a78f292c7a5",
            "value": 74895
          }
        },
        "48058b8aa4c240ae9c5645db5c65f8ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72567652d7a1471294ff05c564e93ad7",
            "placeholder": "​",
            "style": "IPY_MODEL_12866bd620574403a8c8b94fdf538b67",
            "value": " 74.9k/74.9k [00:00&lt;00:00, 1.87MB/s]"
          }
        },
        "5d72b7b4707e404b96017510bb1328f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc9838f7d23b458281aa790d42b77c81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86e612bb868c4accaf07543bffa58d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ec551ed9de94753b9a17e794938eab2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00c73d99ab8c46c096431a78f292c7a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72567652d7a1471294ff05c564e93ad7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12866bd620574403a8c8b94fdf538b67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c8cf500c16f4a3a8345b6fc61184dbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6e60c60a05394708abc01127a65c600b",
              "IPY_MODEL_9f72ed068c43438cb85484f34f4866f8",
              "IPY_MODEL_eb9d6ffe1e8d41cfbac7865cd20b9830"
            ],
            "layout": "IPY_MODEL_881d913ecb4d44b29f49b3e125152dc9"
          }
        },
        "6e60c60a05394708abc01127a65c600b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32d5642ec1a0493c8e8bf9e1870387ee",
            "placeholder": "​",
            "style": "IPY_MODEL_042314e7fa404c54a47012568a3a7067",
            "value": "Generating test split: 100%"
          }
        },
        "9f72ed068c43438cb85484f34f4866f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a2552ac6cbc4bffacc42870b646b21f",
            "max": 56,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2ce310d4137d456d920d388882b09719",
            "value": 56
          }
        },
        "eb9d6ffe1e8d41cfbac7865cd20b9830": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84961f5009414d3fb37fe8871c98f45c",
            "placeholder": "​",
            "style": "IPY_MODEL_e6e4928ee2c04803b6830e92736f2223",
            "value": " 56/56 [00:00&lt;00:00, 565.75 examples/s]"
          }
        },
        "881d913ecb4d44b29f49b3e125152dc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32d5642ec1a0493c8e8bf9e1870387ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "042314e7fa404c54a47012568a3a7067": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a2552ac6cbc4bffacc42870b646b21f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ce310d4137d456d920d388882b09719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "84961f5009414d3fb37fe8871c98f45c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6e4928ee2c04803b6830e92736f2223": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d81af0dcaeeb4cd6a8937fbad0ae02f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7e7ea579720440638aaf14a48d224a66",
              "IPY_MODEL_1fc781ff24e74224b5546a2c897b7220",
              "IPY_MODEL_1791fc4f7ddb480abb79d012486fe6ca"
            ],
            "layout": "IPY_MODEL_07ab60f0178f465ab8ec7b3dfe7fd384"
          }
        },
        "7e7ea579720440638aaf14a48d224a66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc6c9f6f66de4ba389975306afdd1f25",
            "placeholder": "​",
            "style": "IPY_MODEL_10221957f03347af99b6c1d5c0694f6b",
            "value": "Generating train split: 100%"
          }
        },
        "1fc781ff24e74224b5546a2c897b7220": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62966b2d246242fea72ad72b85485c16",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9f2535a0af744b1db11cd0580064543a",
            "value": 100
          }
        },
        "1791fc4f7ddb480abb79d012486fe6ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf960782c33a4f2bbe646844fe1d6880",
            "placeholder": "​",
            "style": "IPY_MODEL_88e29be850ea431aaab4fe21d09400a6",
            "value": " 100/100 [00:00&lt;00:00, 1746.88 examples/s]"
          }
        },
        "07ab60f0178f465ab8ec7b3dfe7fd384": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc6c9f6f66de4ba389975306afdd1f25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10221957f03347af99b6c1d5c0694f6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "62966b2d246242fea72ad72b85485c16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f2535a0af744b1db11cd0580064543a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cf960782c33a4f2bbe646844fe1d6880": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88e29be850ea431aaab4fe21d09400a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}