generate-test-train-data.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/datakind/hxl-metadata-prediction/blob/main/generate-test-train-data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mqLphrzhAizx"
      },
      "source": [
        "# Introduction\n",
        "\n",
        "This notebook downloads and prepares training data for predicing Humanitarian Exchange Language (HXL) metadata tags on humanitarian datasets on the Humanitarian Data Exchange (HDX) platform.\n",
        "\n",
        "This notebook downloads data provided by the HDX team from a google drive folder. The data was captured using an [HXL crawl process](https://github.com/HXLStandard/hdx-hashtag-crawler). It also downloads the [HXL core schema](https://data.humdata.org/dataset/hxl-core-schemas) which defines supported tags and attributes. Finally, the data is split into test and training, being careful to ensure data produced by similar processes and organizations is not present in both test and train.  \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZMyAru65hk3U"
      },
      "source": [
        "# Setup\n",
        "\n",
        "See [README](README.md) from instructions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "WzPWBJEoBC1f"
      },
      "outputs": [],
      "source": [
        "!pip install gdown==5.2.0\n",
        "!pip install pandas==2.2.2\n",
        "!pip install hdx-python-api==6.3.1\n",
        "!pip install openai==1.35.3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_FQ8PBPGxH2",
        "outputId": "8df00eb6-ce01-4fc1-9bdd-9ce370fca123"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DUIEgLN7Aizy"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "import os\n",
        "import requests\n",
        "import gdown\n",
        "import tarfile\n",
        "import pandas as pd\n",
        "import re\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from hdx.utilities.easy_logging import setup_logging\n",
        "from hdx.api.configuration import Configuration\n",
        "from hdx.data.dataset import Dataset\n",
        "import hxl\n",
        "import json\n",
        "import time\n",
        "from openai import OpenAI\n",
        "import numpy as np\n",
        "\n",
        "if os.getenv(\"OPENAI_API_KEY\") is None:\n",
        "    from google.colab import userdata\n",
        "\n",
        "    OPENAI_API_KEY = userdata.get(\"OPENAI_API_KEY\")\n",
        "else:\n",
        "    OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
        "\n",
        "client = OpenAI(api_key=OPENAI_API_KEY)\n",
        "\n",
        "# If using Colab, this is where Google drive gets mounted. Otherwise leave blank\n",
        "GOOGLE_BASE_DIR = \"/content/drive/MyDrive/Colab\"\n",
        "\n",
        "# Where to save local data files\n",
        "LOCAL_DATA_DIR = f\"{GOOGLE_BASE_DIR}/hxl-metadata-prediction/data/\"\n",
        "\n",
        "# Google drive location of HDXHashtag crawler data. Shared with HDX team\n",
        "HXL_CRAWLER_DATA_GDRIVE = \"https://drive.google.com/file/d/1BDCuh0WVJWK1-1RMC-77cvh4H2Hep_ry/export?format=xlsx\"\n",
        "HXL_CRAWLER_DATA_FILE = LOCAL_DATA_DIR + \"/hdx-hxl-output.tgz\"\n",
        "\n",
        "# This is the HXL schema sheet, search HDX to get this link\n",
        "HXL_SCHEMA_RESOURCE_URL = \"https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/export?format=xlsx\"\n",
        "HXL_SCHEMA_LOCAL_FILE = LOCAL_DATA_DIR + \"/hxl-core-schema.xlsx\"\n",
        "\n",
        "# Number of records in data excerpts\n",
        "DATA_EXCERPT_SIZE = 10\n",
        "\n",
        "# Data Summary LLM, used to summarize tabular data for use in prompts\n",
        "# DATA_SUMMARY_LLM = \"gpt-4o-mini\"\n",
        "DATA_SUMMARY_LLM = \"gpt-3.5-turbo\"\n",
        "\n",
        "# Set this to a name of your choosing\n",
        "HDX_USER_AGENT = \"hxl-metadata-prediction\"\n",
        "\n",
        "input_options = hxl.input.InputOptions(http_headers={\"User-Agent\": HDX_USER_AGENT})\n",
        "\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "pd.set_option(\"display.max_rows\", 200)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N-c0JLRVhplK"
      },
      "source": [
        "## Get HDX connection\n",
        "\n",
        "We need a connection to HDX to extract excerpts of data for prompts."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RYSF8CNvAizy",
        "outputId": "a5d1cbf1-8604-4fce-f3aa-e29b6f136030"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Configuration already created, continuing ...\n"
          ]
        }
      ],
      "source": [
        "def setup_hdx_connection(agent_name):\n",
        "    \"\"\"\n",
        "    Set up the HDX connection with the specified agent name.\n",
        "\n",
        "    Parameters:\n",
        "    - agent_name (str): The name of the agent.\n",
        "\n",
        "    Returns:\n",
        "    None\n",
        "    \"\"\"\n",
        "    try:\n",
        "        Configuration.create(hdx_site=\"prod\", user_agent=agent_name, hdx_read_only=True)\n",
        "    except:\n",
        "        print(\"Configuration already created, continuing ...\")\n",
        "\n",
        "\n",
        "# Note, if you run this twice you will get a 'Configuration already exists!' error, but it can be ignored\n",
        "setup_hdx_connection(HDX_USER_AGENT)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1pupY-aehrbo"
      },
      "source": [
        "## Download HXL Core schema\n",
        "\n",
        "We will download the original google sheet defining the HXL core schema, as found on [HDX](https://data.humdata.org/dataset/hxl-core-schemas).\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6DT3fmS9hJMX",
        "outputId": "0782940f-1736-43bb-8aaa-643948325217"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/export?format=xlsx\n",
            "To: /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/hxl-core-schema.xlsx\n",
            "240kB [00:00, 29.9MB/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Approved HXL schema ...\n",
            "['#access', '#activity', '#adm1', '#adm2', '#adm3', '#adm4', '#adm5', '#affected', '#beneficiary', '#capacity', '#cause', '#channel', '#contact', '#country', '#crisis', '#currency', '#date', '#delivery', '#description', '#event', '#frequency', '#geo', '#group', '#impact', '#indicator', '#inneed', '#item', '#loc', '#meta', '#modality', '#need', '#operations', '#org', '#output', '#population', '#reached', '#region', '#respondee', '#sector', '#service', '#severity', '#status', '#subsector', '#targeted', '#value', '+abducted', '+acronym', '+activity', '+adolescents', '+adults', '+approved', '+ar', '+bounds', '+budget', '+canceled', '+children', '+cluster', '+code', '+converted', '+coord', '+dest', '+displaced', '+elderly', '+elevation', '+email', '+en', '+end', '+es', '+f', '+fa', '+fr', '+funder', '+hh', '+i', '+id', '+idps', '+impl', '+incamp', '+ind', '+infants', '+infected', '+injured', '+killed', '+label', '+lat', '+lon', '+m', '+ms', '+name', '+noncamp', '+num', '+occurred', '+origin', '+pct', '+phone', '+prog', '+programme', '+project', '+provided', '+ps', '+refugees', '+reported', '+reporting', '+return', '+returnees', '+source', '+spend', '+start', '+sw', '+text', '+tier', '+tl', '+transfer', '+type', '+uk', '+ur', '+url', '+used']\n"
          ]
        }
      ],
      "source": [
        "gdown.download(HXL_SCHEMA_RESOURCE_URL, HXL_SCHEMA_LOCAL_FILE, quiet=False, fuzzy=True)\n",
        "\n",
        "response = requests.get(HXL_SCHEMA_RESOURCE_URL)\n",
        "with open(HXL_SCHEMA_LOCAL_FILE, \"wb\") as f:\n",
        "    f.write(response.content)\n",
        "\n",
        "df = pd.read_excel(HXL_SCHEMA_LOCAL_FILE, sheet_name=\"Core hashtags\")\n",
        "hashtags_list = df[\"Hashtag\"][1:].tolist()\n",
        "\n",
        "df = pd.read_excel(HXL_SCHEMA_LOCAL_FILE, sheet_name=\"Core attributes\")\n",
        "attributes_list = df[\"Attribute\"][1:].tolist()\n",
        "\n",
        "# Remove rows with disallowed tags or attributes\n",
        "APPROVED_HXL_SCHEMA = hashtags_list + attributes_list\n",
        "\n",
        "print(\"Approved HXL schema ...\")\n",
        "print(APPROVED_HXL_SCHEMA)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4NJSmHplAizy"
      },
      "source": [
        "# Analysis\n",
        "\n",
        "## Download HXL crawler data\n",
        "\n",
        "Many datasets on HDX are created by automatic pipelines. Additionally, organizations will use the same columns and HXL tags for multiple tabular datasets, for example in uploading data for different countries. To avoid polluting the test set with very similar data as found in the training set, we cannot just sample to create a train/test split.\n",
        "Luckily the HDX team have created an MD5 hash for every combination of column names, which we can use to exclude repeat datasets.\n",
        "\n",
        "First, we load the HXL crawler data …\n",
        "\n",
        "This data was generated using the [HDX Hashtag Crawler](https://github.com/dividor/hdx-hashtag-crawler) over several days and saved to Google drive."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "twNj9r0zAizz",
        "outputId": "cfef8ae5-fc05-4799-95a1-8f7e63ba060d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/file/d/1BDCuh0WVJWK1-1RMC-77cvh4H2Hep_ry/export?format=xlsx\n",
            "To: /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/hdx-hxl-output.tgz\n",
            "3.04kB [00:00, 1.38MB/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "HXL core schema saved to /content/drive/MyDrive/Colab/hxl-metadata-prediction/data//hdx-hxl-output.tgz\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "gdown.download(HXL_CRAWLER_DATA_GDRIVE, HXL_CRAWLER_DATA_FILE, quiet=False, fuzzy=True)\n",
        "print(f\"HXL core schema saved to {HXL_CRAWLER_DATA_FILE}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xLszv6-4Aizz"
      },
      "source": [
        "## Identify Unique combinations of HXL tags we want for training data\n",
        "\n",
        "The HXL crawler and reports data create a hash based on column names, referred to as a 'Resource pattern' by the HDX team. This is useful because a lot of HDX resources (tables) are generated by automatic pipeline, so rather than duplicating very similar data in our training set, we will focus on unique combinations of column headers. This is an indirect way of balancing the data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 449
        },
        "id": "0nDIHJSDAizz",
        "outputId": "da3ea0b5-56aa-4db7-d318-a5be61c443b0"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-17-194518c6290b>:1: DtypeWarning: Columns (9) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(LOCAL_DATA_DIR + \"/output/hdx-expanded-hashed-stats.csv\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of rows in data ...\n",
            "(487297, 9)\n",
            "Unique data providers ...\n",
            "120\n",
            "Unique HDX resource ids ...\n",
            "43074\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "repr_error": "0",
              "type": "dataframe"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-450332ac-8909-4956-8ccd-81392663d635\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "      <th>Text header</th>\n",
              "      <th>Locations</th>\n",
              "      <th>Data provider</th>\n",
              "      <th>HDX dataset id</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Date created</th>\n",
              "      <th>Unnamed: 9</th>\n",
              "      <th>Hash</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>#affected+hh</td>\n",
              "      <td>Total IDP HH</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>#affected+idp+ind</td>\n",
              "      <td>Total IDP IND</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>#affected+idp+male</td>\n",
              "      <td>Total IDP Male Ind</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>#affected+female+idp</td>\n",
              "      <td>Total IDP Female Ind</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>#affected+ind+returnees</td>\n",
              "      <td>Total Returnees</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-450332ac-8909-4956-8ccd-81392663d635')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-450332ac-8909-4956-8ccd-81392663d635 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-450332ac-8909-4956-8ccd-81392663d635');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-01b8b54c-c8e9-48bb-9fc6-d45f52df61fe\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-01b8b54c-c8e9-48bb-9fc6-d45f52df61fe')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-01b8b54c-c8e9-48bb-9fc6-d45f52df61fe button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "   Hashtag with Attributes           Text header Locations  \\\n",
              "1             #affected+hh          Total IDP HH       COD   \n",
              "2        #affected+idp+ind         Total IDP IND       COD   \n",
              "4       #affected+idp+male    Total IDP Male Ind       COD   \n",
              "6     #affected+female+idp  Total IDP Female Ind       COD   \n",
              "8  #affected+ind+returnees       Total Returnees       COD   \n",
              "\n",
              "                              Data provider  \\\n",
              "1  international-organization-for-migration   \n",
              "2  international-organization-for-migration   \n",
              "4  international-organization-for-migration   \n",
              "6  international-organization-for-migration   \n",
              "8  international-organization-for-migration   \n",
              "\n",
              "                                                                               HDX dataset id  \\\n",
              "1  drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "2  drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "4  drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "6  drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "8  drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "\n",
              "                        HDX resource id Date created Unnamed: 9  \\\n",
              "1  26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "2  26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "4  26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "6  26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "8  26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "\n",
              "                 Hash  \n",
              "1  0x2cc7fd3129c0d18c  \n",
              "2  0x2cc7fd3129c0d18c  \n",
              "4  0x2cc7fd3129c0d18c  \n",
              "6  0x2cc7fd3129c0d18c  \n",
              "8  0x2cc7fd3129c0d18c  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "df = pd.read_csv(LOCAL_DATA_DIR + \"/output/hdx-expanded-hashed-stats.csv\")\n",
        "\n",
        "# We'll keep one row per column, Hashtag with Attributes has what we require\n",
        "df.drop(columns=[\"Attribute\", \"Hashtag\"], inplace=True)\n",
        "df.drop_duplicates(inplace=True)\n",
        "\n",
        "# Remove HXL tags row in the metadata (we keep them for actual data)\n",
        "df = df[1:]\n",
        "\n",
        "print(\"Number of rows in data ...\")\n",
        "print(df.shape)\n",
        "\n",
        "print(\"Unique data providers ...\")\n",
        "print(len(df[\"Data provider\"].unique()))\n",
        "\n",
        "print(\"Unique HDX resource ids ...\")\n",
        "print(len(df[\"HDX resource id\"].unique()))\n",
        "\n",
        "display(df.head())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SL_jdFaRAizz"
      },
      "source": [
        "Let's use the column hash created by the crawler to find unique combinations of tags"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xmwU9aCiAizz"
      },
      "outputs": [],
      "source": [
        "hash_count = df.groupby(\"Hash\").size()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 631
        },
        "id": "4mBEyCY9Aizz",
        "outputId": "58982f7b-1de6-4f6f-cabe-d8cf894fe620"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"hash_resources\",\n  \"rows\": 644,\n  \"fields\": [\n    {\n      \"column\": \"level_0\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 186,\n        \"min\": 0,\n        \"max\": 643,\n        \"num_unique_values\": 644,\n        \"samples\": [\n          631,\n          592,\n          599\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"index\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 186,\n        \"min\": 0,\n        \"max\": 643,\n        \"num_unique_values\": 644,\n        \"samples\": [\n          631,\n          592,\n          599\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Hash\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 644,\n        \"samples\": [\n          \"0x4ef17b2ac68bfa6a\",\n          \"0x58e276d480fdb484\",\n          \"0x1dd642e15d731daa\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"HDX resource id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 694,\n        \"min\": 1,\n        \"max\": 13858,\n        \"num_unique_values\": 70,\n        \"samples\": [\n          139,\n          13858,\n          24\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unique HDX resource id\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unique HDX dataset id\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unique Data provider\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe",
              "variable_name": "hash_resources"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-67c18dad-8e8a-48a7-a56d-defa9758e56a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>level_0</th>\n",
              "      <th>index</th>\n",
              "      <th>Hash</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Unique HDX resource id</th>\n",
              "      <th>Unique HDX dataset id</th>\n",
              "      <th>Unique Data provider</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0x102125f4dd16c64</td>\n",
              "      <td>13858</td>\n",
              "      <td>[51b2e4ec-aca5-4b97-bbb7-c005175b682e, a6ef8040-3b15-47ae-9973-1dbc113673cf, f579cf0e-5535-4414-897f-2f8c05105180, 66c62464-017b-4aa6-845f-9ec2487acb82, 91e1cb98-353b-487e-a14d-b0eea783da6f, 65f38...</td>\n",
              "      <td>[who-data-for-south-sudan, who-data-for-montenegro, who-data-for-zimbabwe, who-data-for-zambia, who-data-for-yemen, who-data-for-viet-nam, who-data-for-venezuela-bolivarian-republic-of, who-data-f...</td>\n",
              "      <td>[world-health-organization]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0x428a8e37940223d8</td>\n",
              "      <td>9328</td>\n",
              "      <td>[2e130cdf-c850-4533-b2f3-e961adbec48a, 9e160d82-691d-49a6-979b-0ff0dbb6b7a8, 002501bc-7efb-4335-b672-d045cd76bc5b, 0d0e0fc4-e4f1-49cd-ad30-42cc8dc08b74, f2c1ea93-d241-413b-8140-c009da88d912, 23e36...</td>\n",
              "      <td>[world-bank-combined-indicators-for-zimbabwe, world-bank-trade-indicators-for-zimbabwe, world-bank-external-debt-indicators-for-zimbabwe, world-bank-climate-change-indicators-for-zimbabwe, world-b...</td>\n",
              "      <td>[world-bank-group]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0x31fda1ef985b4a59</td>\n",
              "      <td>3425</td>\n",
              "      <td>[9c751883-698a-4a2c-9475-ff828e9c11db, 791b69af-df57-4157-96c0-c0d8d308315e, b055f1f7-8cdc-4ff8-a412-9f54d6e56c41, 6ae8568f-449b-4b59-9ad2-79d7df94cd9c, 4f4c7462-6b13-4125-a88a-f8e9ac0c837b, abb79...</td>\n",
              "      <td>[dhs-data-for-sao-tome-and-principe, dhs-data-for-rwanda, dhs-data-for-philippines, dhs-data-for-peru, dhs-data-for-paraguay, dhs-data-for-papua-new-guinea, dhs-data-for-pakistan, dhs-data-for-nig...</td>\n",
              "      <td>[dhs]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0x19598575d3397e19</td>\n",
              "      <td>3232</td>\n",
              "      <td>[ed7b5bd2-7818-4d7a-9ff0-8ba0d97bf7d5, 8f239e93-76c0-4287-a414-3d17a5e55344, 040efd19-3d71-4e0b-8939-f6b46c465868, 8db41d1c-af8f-4f26-9d40-5b80b1bf72e8, 6482dc75-0e79-40d9-aacf-cdcde3c368a6, c9981...</td>\n",
              "      <td>[dhs-subnational-data-for-sao-tome-and-principe, dhs-subnational-data-for-rwanda, dhs-subnational-data-for-philippines, dhs-subnational-data-for-peru, dhs-subnational-data-for-paraguay, dhs-subnat...</td>\n",
              "      <td>[dhs]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>0x16d2b679132fea10</td>\n",
              "      <td>2147</td>\n",
              "      <td>[295cd9e4-8464-43ee-ad17-47196991a1f7, 34337d16-017d-4d69-834c-a5e0fc21a549, b51b8c0e-494d-488b-98d5-a70fd9451b90, 75076d6a-8d3f-49e3-b4f4-7c889bc82806, 7e0b2b37-73b8-4c69-bb11-6ba954fa0cd9, 08ea0...</td>\n",
              "      <td>[unhcr-population-data-for-world, unhcr-population-data-for-zwe, unhcr-population-data-for-zmb, unhcr-population-data-for-zaf, unhcr-population-data-for-yem, unhcr-population-data-for-wsm, unhcr-p...</td>\n",
              "      <td>[unhcr]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>639</th>\n",
              "      <td>639</td>\n",
              "      <td>639</td>\n",
              "      <td>0x22def8e1b7b0c742</td>\n",
              "      <td>1</td>\n",
              "      <td>[68328f42-9276-423e-80d0-fe89630804ff]</td>\n",
              "      <td>[3w-december-2017]</td>\n",
              "      <td>[ocha-ethiopia]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>640</th>\n",
              "      <td>640</td>\n",
              "      <td>640</td>\n",
              "      <td>0x4b7012601f2de402</td>\n",
              "      <td>1</td>\n",
              "      <td>[63143067-46e0-4fb5-b131-d83ee45122ab]</td>\n",
              "      <td>[ethiopia-settlements]</td>\n",
              "      <td>[ocha-ethiopia]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>641</th>\n",
              "      <td>641</td>\n",
              "      <td>641</td>\n",
              "      <td>0x4aefd35864aaa20a</td>\n",
              "      <td>1</td>\n",
              "      <td>[6ddcfbc9-fa06-4b14-b9a4-ce96d3fae65e]</td>\n",
              "      <td>[base-acceso-internet-personas-entre-los-5-y-19-anos-2018]</td>\n",
              "      <td>[immap]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>642</th>\n",
              "      <td>642</td>\n",
              "      <td>642</td>\n",
              "      <td>0x4acf4e36d67d877</td>\n",
              "      <td>1</td>\n",
              "      <td>[903326f2-b372-4786-9973-87226cb15e41]</td>\n",
              "      <td>[people-in-need-2008-2019]</td>\n",
              "      <td>[ocha-fts]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>643</th>\n",
              "      <td>643</td>\n",
              "      <td>643</td>\n",
              "      <td>0x57297f722f5115a5</td>\n",
              "      <td>1</td>\n",
              "      <td>[8b0feea6-20e4-45dc-aaae-c8b6fbd5a9f4]</td>\n",
              "      <td>[burkina-faso-plan-d-urgence]</td>\n",
              "      <td>[ocha-rowca]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>644 rows × 7 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-67c18dad-8e8a-48a7-a56d-defa9758e56a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-67c18dad-8e8a-48a7-a56d-defa9758e56a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-67c18dad-8e8a-48a7-a56d-defa9758e56a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-294176a3-f7d0-4e5d-8441-8b8ac6b096c3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-294176a3-f7d0-4e5d-8441-8b8ac6b096c3')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-294176a3-f7d0-4e5d-8441-8b8ac6b096c3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_b4a86f9b-7d99-4331-beda-2d1d6bac7322\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('hash_resources')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_b4a86f9b-7d99-4331-beda-2d1d6bac7322 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('hash_resources');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "     level_0  index                Hash  HDX resource id  \\\n",
              "0          0      0   0x102125f4dd16c64            13858   \n",
              "1          1      1  0x428a8e37940223d8             9328   \n",
              "2          2      2  0x31fda1ef985b4a59             3425   \n",
              "3          3      3  0x19598575d3397e19             3232   \n",
              "4          4      4  0x16d2b679132fea10             2147   \n",
              "..       ...    ...                 ...              ...   \n",
              "639      639    639  0x22def8e1b7b0c742                1   \n",
              "640      640    640  0x4b7012601f2de402                1   \n",
              "641      641    641  0x4aefd35864aaa20a                1   \n",
              "642      642    642   0x4acf4e36d67d877                1   \n",
              "643      643    643  0x57297f722f5115a5                1   \n",
              "\n",
              "                                                                                                                                                                                      Unique HDX resource id  \\\n",
              "0    [51b2e4ec-aca5-4b97-bbb7-c005175b682e, a6ef8040-3b15-47ae-9973-1dbc113673cf, f579cf0e-5535-4414-897f-2f8c05105180, 66c62464-017b-4aa6-845f-9ec2487acb82, 91e1cb98-353b-487e-a14d-b0eea783da6f, 65f38...   \n",
              "1    [2e130cdf-c850-4533-b2f3-e961adbec48a, 9e160d82-691d-49a6-979b-0ff0dbb6b7a8, 002501bc-7efb-4335-b672-d045cd76bc5b, 0d0e0fc4-e4f1-49cd-ad30-42cc8dc08b74, f2c1ea93-d241-413b-8140-c009da88d912, 23e36...   \n",
              "2    [9c751883-698a-4a2c-9475-ff828e9c11db, 791b69af-df57-4157-96c0-c0d8d308315e, b055f1f7-8cdc-4ff8-a412-9f54d6e56c41, 6ae8568f-449b-4b59-9ad2-79d7df94cd9c, 4f4c7462-6b13-4125-a88a-f8e9ac0c837b, abb79...   \n",
              "3    [ed7b5bd2-7818-4d7a-9ff0-8ba0d97bf7d5, 8f239e93-76c0-4287-a414-3d17a5e55344, 040efd19-3d71-4e0b-8939-f6b46c465868, 8db41d1c-af8f-4f26-9d40-5b80b1bf72e8, 6482dc75-0e79-40d9-aacf-cdcde3c368a6, c9981...   \n",
              "4    [295cd9e4-8464-43ee-ad17-47196991a1f7, 34337d16-017d-4d69-834c-a5e0fc21a549, b51b8c0e-494d-488b-98d5-a70fd9451b90, 75076d6a-8d3f-49e3-b4f4-7c889bc82806, 7e0b2b37-73b8-4c69-bb11-6ba954fa0cd9, 08ea0...   \n",
              "..                                                                                                                                                                                                       ...   \n",
              "639                                                                                                                                                                   [68328f42-9276-423e-80d0-fe89630804ff]   \n",
              "640                                                                                                                                                                   [63143067-46e0-4fb5-b131-d83ee45122ab]   \n",
              "641                                                                                                                                                                   [6ddcfbc9-fa06-4b14-b9a4-ce96d3fae65e]   \n",
              "642                                                                                                                                                                   [903326f2-b372-4786-9973-87226cb15e41]   \n",
              "643                                                                                                                                                                   [8b0feea6-20e4-45dc-aaae-c8b6fbd5a9f4]   \n",
              "\n",
              "                                                                                                                                                                                       Unique HDX dataset id  \\\n",
              "0    [who-data-for-south-sudan, who-data-for-montenegro, who-data-for-zimbabwe, who-data-for-zambia, who-data-for-yemen, who-data-for-viet-nam, who-data-for-venezuela-bolivarian-republic-of, who-data-f...   \n",
              "1    [world-bank-combined-indicators-for-zimbabwe, world-bank-trade-indicators-for-zimbabwe, world-bank-external-debt-indicators-for-zimbabwe, world-bank-climate-change-indicators-for-zimbabwe, world-b...   \n",
              "2    [dhs-data-for-sao-tome-and-principe, dhs-data-for-rwanda, dhs-data-for-philippines, dhs-data-for-peru, dhs-data-for-paraguay, dhs-data-for-papua-new-guinea, dhs-data-for-pakistan, dhs-data-for-nig...   \n",
              "3    [dhs-subnational-data-for-sao-tome-and-principe, dhs-subnational-data-for-rwanda, dhs-subnational-data-for-philippines, dhs-subnational-data-for-peru, dhs-subnational-data-for-paraguay, dhs-subnat...   \n",
              "4    [unhcr-population-data-for-world, unhcr-population-data-for-zwe, unhcr-population-data-for-zmb, unhcr-population-data-for-zaf, unhcr-population-data-for-yem, unhcr-population-data-for-wsm, unhcr-p...   \n",
              "..                                                                                                                                                                                                       ...   \n",
              "639                                                                                                                                                                                       [3w-december-2017]   \n",
              "640                                                                                                                                                                                   [ethiopia-settlements]   \n",
              "641                                                                                                                                               [base-acceso-internet-personas-entre-los-5-y-19-anos-2018]   \n",
              "642                                                                                                                                                                               [people-in-need-2008-2019]   \n",
              "643                                                                                                                                                                            [burkina-faso-plan-d-urgence]   \n",
              "\n",
              "            Unique Data provider  \n",
              "0    [world-health-organization]  \n",
              "1             [world-bank-group]  \n",
              "2                          [dhs]  \n",
              "3                          [dhs]  \n",
              "4                        [unhcr]  \n",
              "..                           ...  \n",
              "639              [ocha-ethiopia]  \n",
              "640              [ocha-ethiopia]  \n",
              "641                      [immap]  \n",
              "642                   [ocha-fts]  \n",
              "643                 [ocha-rowca]  \n",
              "\n",
              "[644 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "hash_resources = (\n",
        "    df.groupby(\"Hash\")[\"HDX resource id\"].nunique().sort_values(ascending=False)\n",
        ")\n",
        "\n",
        "for col in [\"HDX resource id\", \"HDX dataset id\", \"Data provider\"]:\n",
        "    hash_resources = hash_resources.reset_index()\n",
        "    hash_resources[f\"Unique {col}\"] = hash_resources[\"Hash\"].map(\n",
        "        df.groupby(\"Hash\")[col].unique()\n",
        "    )\n",
        "\n",
        "display(hash_resources)\n",
        "\n",
        "hash_resources.to_excel(f\"{LOCAL_DATA_DIR}/hxl_hash_resources.xlsx\", index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note how a data provider will often contribute multiple resource ids (CSV and Excel tables), all with the same column names. There isn't much benefit in including this repeat data in the model fine-tuning training, but we do want one representative example for each column combination (hash) …"
      ],
      "metadata": {
        "id": "_nJOt0NiV8i6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6b2U9ChOAizz",
        "outputId": "81a5d9e9-cadd-4895-fca9-0a5f5cf5f124"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0      51b2e4ec-aca5-4b97-bbb7-c005175b682e\n",
            "1      2e130cdf-c850-4533-b2f3-e961adbec48a\n",
            "2      9c751883-698a-4a2c-9475-ff828e9c11db\n",
            "3      ed7b5bd2-7818-4d7a-9ff0-8ba0d97bf7d5\n",
            "4      295cd9e4-8464-43ee-ad17-47196991a1f7\n",
            "                       ...                 \n",
            "639    68328f42-9276-423e-80d0-fe89630804ff\n",
            "640    63143067-46e0-4fb5-b131-d83ee45122ab\n",
            "641    6ddcfbc9-fa06-4b14-b9a4-ce96d3fae65e\n",
            "642    903326f2-b372-4786-9973-87226cb15e41\n",
            "643    8b0feea6-20e4-45dc-aaae-c8b6fbd5a9f4\n",
            "Name: Unique HDX resource id, Length: 644, dtype: object\n"
          ]
        }
      ],
      "source": [
        "# Extract a single resource_id for each hash\n",
        "resource_ids = hash_resources[\"Unique HDX resource id\"].apply(lambda x: x[0])\n",
        "print(resource_ids[0:5])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QHXfoCBvAizz",
        "outputId": "d0b16c98-6a33-4224-cd65-ad0ed0a66a75"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Column data subset to one resource ID per hash ...\n",
            "(7834, 9)\n",
            "Unique data providers ...\n",
            "119\n",
            "Unique HDX resource ids ...\n",
            "644\n"
          ]
        }
      ],
      "source": [
        "df_subset = df[df[\"HDX resource id\"].isin(resource_ids)]\n",
        "\n",
        "print(\"Column data subset to one resource ID per hash ...\")\n",
        "print(df_subset.shape)\n",
        "\n",
        "print(\"Unique data providers ...\")\n",
        "print(len(df_subset[\"Data provider\"].unique()))\n",
        "\n",
        "print(\"Unique HDX resource ids ...\")\n",
        "print(len(df_subset[\"HDX resource id\"].unique()))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This has reduced our data significantly, but in doing so we have indirectly performed some balancing."
      ],
      "metadata": {
        "id": "T7LtL-IMWFhY"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vJkW8wyML2OI"
      },
      "source": [
        "## Remove unsupported HXL tags and attributes\n",
        "\n",
        "Next, we will remove HXL tags and attributes which are not officially supported in the HXL Core schema."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 666
        },
        "id": "ZBTyYEXCL7WD",
        "outputId": "c77c549d-9ce6-4e07-a316-9016d21b2863"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Before (7834, 9)\n",
            "After (3777, 9)\n",
            "(3777, 9)\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "repr_error": "0",
              "type": "dataframe",
              "variable_name": "disallowed"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-0cb0c1e1-4cd6-40df-8c44-672fa67bb6df\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "      <th>Text header</th>\n",
              "      <th>Locations</th>\n",
              "      <th>Data provider</th>\n",
              "      <th>HDX dataset id</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Date created</th>\n",
              "      <th>Unnamed: 9</th>\n",
              "      <th>Hash</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>#affected+idp+ind</td>\n",
              "      <td>Total IDP IND</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>#affected+idp+male</td>\n",
              "      <td>Total IDP Male Ind</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>#affected+female+idp</td>\n",
              "      <td>Total IDP Female Ind</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>256</th>\n",
              "      <td>#meta+appeal+type</td>\n",
              "      <td>atype</td>\n",
              "      <td>SSD</td>\n",
              "      <td>ifrc</td>\n",
              "      <td>ifrc-appeals-data-for-south-sudan</td>\n",
              "      <td>4110b824-3338-453f-ae5d-89ca80f5b147</td>\n",
              "      <td>2023-03-13</td>\n",
              "      <td>true</td>\n",
              "      <td>0x1d1434ee319a1be</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>260</th>\n",
              "      <td>#meta+appeal+id</td>\n",
              "      <td>code</td>\n",
              "      <td>SSD</td>\n",
              "      <td>ifrc</td>\n",
              "      <td>ifrc-appeals-data-for-south-sudan</td>\n",
              "      <td>4110b824-3338-453f-ae5d-89ca80f5b147</td>\n",
              "      <td>2023-03-13</td>\n",
              "      <td>true</td>\n",
              "      <td>0x1d1434ee319a1be</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689570</th>\n",
              "      <td>#lat_deg</td>\n",
              "      <td>prevlat</td>\n",
              "      <td>VUT</td>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>cyclone-pam-path</td>\n",
              "      <td>a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3</td>\n",
              "      <td>2015-03-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x1d4a8deeb40f76ce</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689571</th>\n",
              "      <td>#lon_deg</td>\n",
              "      <td>prevlon</td>\n",
              "      <td>VUT</td>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>cyclone-pam-path</td>\n",
              "      <td>a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3</td>\n",
              "      <td>2015-03-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x1d4a8deeb40f76ce</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689572</th>\n",
              "      <td>#period_date</td>\n",
              "      <td>datelabel</td>\n",
              "      <td>VUT</td>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>cyclone-pam-path</td>\n",
              "      <td>a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3</td>\n",
              "      <td>2015-03-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x1d4a8deeb40f76ce</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689573</th>\n",
              "      <td>#x_time</td>\n",
              "      <td>hours</td>\n",
              "      <td>VUT</td>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>cyclone-pam-path</td>\n",
              "      <td>a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3</td>\n",
              "      <td>2015-03-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x1d4a8deeb40f76ce</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689574</th>\n",
              "      <td>#x_note</td>\n",
              "      <td>text</td>\n",
              "      <td>VUT</td>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>cyclone-pam-path</td>\n",
              "      <td>a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3</td>\n",
              "      <td>2015-03-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x1d4a8deeb40f76ce</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4057 rows × 9 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0cb0c1e1-4cd6-40df-8c44-672fa67bb6df')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0cb0c1e1-4cd6-40df-8c44-672fa67bb6df button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0cb0c1e1-4cd6-40df-8c44-672fa67bb6df');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-68196041-32a0-49e6-874e-698a02c90874\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-68196041-32a0-49e6-874e-698a02c90874')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-68196041-32a0-49e6-874e-698a02c90874 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_a1af2a08-e304-4fe6-9622-1b73578e9aee\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('disallowed')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_a1af2a08-e304-4fe6-9622-1b73578e9aee button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('disallowed');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "       Hashtag with Attributes           Text header Locations  \\\n",
              "2            #affected+idp+ind         Total IDP IND       COD   \n",
              "4           #affected+idp+male    Total IDP Male Ind       COD   \n",
              "6         #affected+female+idp  Total IDP Female Ind       COD   \n",
              "256          #meta+appeal+type                 atype       SSD   \n",
              "260            #meta+appeal+id                  code       SSD   \n",
              "...                        ...                   ...       ...   \n",
              "689570                #lat_deg               prevlat       VUT   \n",
              "689571                #lon_deg               prevlon       VUT   \n",
              "689572            #period_date             datelabel       VUT   \n",
              "689573                 #x_time                 hours       VUT   \n",
              "689574                 #x_note                  text       VUT   \n",
              "\n",
              "                                   Data provider  \\\n",
              "2       international-organization-for-migration   \n",
              "4       international-organization-for-migration   \n",
              "6       international-organization-for-migration   \n",
              "256                                         ifrc   \n",
              "260                                         ifrc   \n",
              "...                                          ...   \n",
              "689570                               brcmapsteam   \n",
              "689571                               brcmapsteam   \n",
              "689572                               brcmapsteam   \n",
              "689573                               brcmapsteam   \n",
              "689574                               brcmapsteam   \n",
              "\n",
              "                                                                                    HDX dataset id  \\\n",
              "2       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "4       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "6       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "256                                                              ifrc-appeals-data-for-south-sudan   \n",
              "260                                                              ifrc-appeals-data-for-south-sudan   \n",
              "...                                                                                            ...   \n",
              "689570                                                                            cyclone-pam-path   \n",
              "689571                                                                            cyclone-pam-path   \n",
              "689572                                                                            cyclone-pam-path   \n",
              "689573                                                                            cyclone-pam-path   \n",
              "689574                                                                            cyclone-pam-path   \n",
              "\n",
              "                             HDX resource id Date created Unnamed: 9  \\\n",
              "2       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "4       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "6       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "256     4110b824-3338-453f-ae5d-89ca80f5b147   2023-03-13       true   \n",
              "260     4110b824-3338-453f-ae5d-89ca80f5b147   2023-03-13       true   \n",
              "...                                      ...          ...        ...   \n",
              "689570  a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3   2015-03-16       True   \n",
              "689571  a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3   2015-03-16       True   \n",
              "689572  a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3   2015-03-16       True   \n",
              "689573  a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3   2015-03-16       True   \n",
              "689574  a8ccd9d2-8328-487a-b04b-ca3f3f2e0ea3   2015-03-16       True   \n",
              "\n",
              "                      Hash  \n",
              "2       0x2cc7fd3129c0d18c  \n",
              "4       0x2cc7fd3129c0d18c  \n",
              "6       0x2cc7fd3129c0d18c  \n",
              "256      0x1d1434ee319a1be  \n",
              "260      0x1d1434ee319a1be  \n",
              "...                    ...  \n",
              "689570  0x1d4a8deeb40f76ce  \n",
              "689571  0x1d4a8deeb40f76ce  \n",
              "689572  0x1d4a8deeb40f76ce  \n",
              "689573  0x1d4a8deeb40f76ce  \n",
              "689574  0x1d4a8deeb40f76ce  \n",
              "\n",
              "[4057 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def filter_for_schema(text):\n",
        "    \"\"\"\n",
        "    Filters the input text to remove spaces and extract approved HXL schema tokens.\n",
        "\n",
        "    Args:\n",
        "        text (str): The input text to be filtered.\n",
        "\n",
        "    Returns:\n",
        "        str: The filtered text containing only approved HXL schema tokens.\n",
        "\n",
        "    \"\"\"\n",
        "    # print(f\"Tokens before: {text}\")\n",
        "    if \" \" in text:\n",
        "        text = text.replace(\" \", \"\")\n",
        "\n",
        "    tokens_raw = text.split(\"+\")\n",
        "    tokens = [tokens_raw[0]]\n",
        "    for t in tokens_raw[1:]:\n",
        "        tokens.append(f\"+{t}\")\n",
        "\n",
        "    filtered = []\n",
        "    for t in tokens:\n",
        "        if t in APPROVED_HXL_SCHEMA:\n",
        "            if t not in filtered:\n",
        "                filtered.append(t)\n",
        "    filtered = \"\".join(filtered)\n",
        "\n",
        "    if len(filtered) > 0 and filtered[0] != \"#\":\n",
        "        filtered = \"\"\n",
        "\n",
        "    return filtered\n",
        "\n",
        "\n",
        "def filter_disallowed_hxl(column_data, hxl_col=\"Hashtag with Attributes\"):\n",
        "    \"\"\"\n",
        "    Filter the column_data DataFrame based on a specific condition.\n",
        "\n",
        "    Args:\n",
        "        column_data (pd.DataFrame): The DataFrame containing the column data.\n",
        "        hxl_col (str, optional): The name of the column to filter on. Defaults to 'Hashtag with Attributes'.\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: The filtered DataFrame containing the allowed rows.\n",
        "        pd.DataFrame: The filtered DataFrame containing the disallowed rows.\n",
        "    \"\"\"\n",
        "    print(\"Before\", column_data.shape)\n",
        "    allowed = []\n",
        "    disallowed = []\n",
        "    for index, row in column_data.iterrows():\n",
        "        if row[hxl_col] == filter_for_schema(row[hxl_col]):\n",
        "            allowed.append(row)\n",
        "        else:\n",
        "            disallowed.append(row)\n",
        "    allowed = pd.DataFrame(allowed)\n",
        "    disallowed = pd.DataFrame(disallowed)\n",
        "    print(\"After\", allowed.shape)\n",
        "    return allowed, disallowed\n",
        "\n",
        "\n",
        "data, disallowed = filter_disallowed_hxl(df_subset)\n",
        "print(data.shape)\n",
        "\n",
        "display(disallowed)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "That's a pretty big drop, indicating that it's challenging for humans to use all the correct tags and attributes when they tag columns.\n",
        "\n",
        "Spot checking a few …\n"
      ],
      "metadata": {
        "id": "g2oNW2WRWLbq"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 711
        },
        "id": "EMJZnf8wOi0D",
        "outputId": "d4c02869-8d22-49c9-892f-8230258018ac"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>653446</th>\n",
              "      <td>#admin1+code</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689561</th>\n",
              "      <td>#period_date</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>632530</th>\n",
              "      <td>#indicator+num+student</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>611944</th>\n",
              "      <td>#value+cerf+funding+gm0+total+usd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>687700</th>\n",
              "      <td>#affected+date+w5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>653131</th>\n",
              "      <td>#value+sanitation5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95582</th>\n",
              "      <td>#indicator+educationfacility+entry+num</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>600075</th>\n",
              "      <td>#loc+name+port</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>612658</th>\n",
              "      <td>#affected+ind+migrant</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689562</th>\n",
              "      <td>#x_time</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689110</th>\n",
              "      <td>#impact+affected+km+roads</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>619402</th>\n",
              "      <td>#group+i_en+name</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>365852</th>\n",
              "      <td>#inneed+boys+gbv</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>515428</th>\n",
              "      <td>#sector+inneed+shelter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>673951</th>\n",
              "      <td>#inneed+rrr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>675336</th>\n",
              "      <td>#sector+i_fr+name+preferred</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>636163</th>\n",
              "      <td>#org+capacity+fsac</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>520069</th>\n",
              "      <td>#severity+health</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689178</th>\n",
              "      <td>#date+year</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>653514</th>\n",
              "      <td>#severity+idps+outcamp</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><br><label><b>dtype:</b> object</label>"
            ],
            "text/plain": [
              "653446                              #admin1+code\n",
              "689561                              #period_date\n",
              "632530                    #indicator+num+student\n",
              "611944         #value+cerf+funding+gm0+total+usd\n",
              "687700                         #affected+date+w5\n",
              "653131                        #value+sanitation5\n",
              "95582     #indicator+educationfacility+entry+num\n",
              "600075                            #loc+name+port\n",
              "612658                     #affected+ind+migrant\n",
              "689562                                   #x_time\n",
              "689110                 #impact+affected+km+roads\n",
              "619402                          #group+i_en+name\n",
              "365852                          #inneed+boys+gbv\n",
              "515428                    #sector+inneed+shelter\n",
              "673951                               #inneed+rrr\n",
              "675336               #sector+i_fr+name+preferred\n",
              "636163                        #org+capacity+fsac\n",
              "520069                          #severity+health\n",
              "689178                                #date+year\n",
              "653514                    #severity+idps+outcamp\n",
              "Name: Hashtag with Attributes, dtype: object"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "disallowed_sample = disallowed.sample(20)\n",
        "display(disallowed_sample[\"Hashtag with Attributes\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Comparing against the [HXL Core Schema](https://data.humdata.org/dataset/hxl-core-schemas), we do in fact see that each has at least one tag or attribute which isn't in the schema list.\n",
        "\n",
        "It might seem a bit aggressive to remove all instances even if one attribute is incorrect, and future work might relax this a bit, but for now to keep things as clean as possible we'll only use data with fully supported tags and attributes.\n"
      ],
      "metadata": {
        "id": "55eU_XuIWRZu"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EvcvYCfbAizz"
      },
      "source": [
        "## Download data excerpts\n",
        "\n",
        "As a (part-time!) human, to set the right tags and attributes on a column in a table I like to see an example of the data in that columns. As shown in my previous blog post, LLMs are the same and perform better if the prompt contains a sneak peek of the data.\n",
        "\n",
        "To get the data excerpts for each column, we can use the HDX Python API along with libhxl. The code below isn't very elegant, future work might add the excerpt extraction to the HXL data crawler perhaps, but it will download tabular datasets and load them to extract data excerpts …"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "collapsed": true,
        "id": "ZXj8g8eZAizz",
        "outputId": "675db250-de0b-4b6c-d47a-ec924d95ecb6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "For each resource, extract a data excerpt for each column ...\n",
            "Processing 0 of 612 (0.00%) resources\n",
            "    Accessing data for resource 26ecc26f-74e7-46af-b450-8872dca0b63b, DRC - Baseline Assessment - M23 Crisis 13 - February 2024\n",
            "    Accessing data for resource dbf9b4bd-1321-4846-b6f0-4654509d3626, admin1-summaries-earthquake.csv\n",
            "    Accessing data for resource 4110b824-3338-453f-ae5d-89ca80f5b147, IFRC Appeals Data for South Sudan\n",
            "    Accessing data for resource b4e5634e-42e0-4ca5-8893-8e15cad6b620, fts_requirements_funding_pse.csv\n",
            "    Accessing data for resource 4854ed70-56fb-4792-8845-bdeb51418c66, fts_incoming_funding_vnm.csv\n",
            "    Accessing data for resource 7b373ee7-af33-4140-92a0-83d2693eb74c, CERF Donor Contributions (HXLated).csv\n",
            "    Accessing data for resource 636c8890-8747-442b-87c9-895133314dcb, idmc event data for WSM\n",
            "    Accessing data for resource 3a53c43d-83bc-46e7-b66a-38ccca4b536c, AWSD_SD_security_incidents.csv\n",
            "    Accessing data for resource 0908c976-8b62-4a7e-9a38-975011baab1e, 2016-2023 SDN Insecurity Insight Data.xlsx\n",
            "Processing 10 of 612 (1.63%) resources\n",
            "    Accessing data for resource bb26f89b-709f-4efb-8f7d-b20c53bfeaa3, DTM Sudan - Weekly displacement snapshot 21\n",
            "    Accessing data for resource ba91fb45-04e2-4742-a32f-47748856f6cb, Floods Data 2021 to 2023.xlsx\n",
            "    Accessing data for resource 71b99412-e3e3-4ee7-895b-1f53f65d8d08, Statistics_NGR_20231231.xlsx\n",
            "    Accessing data for resource ed7b5bd2-7818-4d7a-9ff0-8ba0d97bf7d5, DHS Quickstats Data for Sao Tome and Principe\n",
            "    Accessing data for resource 9c751883-698a-4a2c-9475-ff828e9c11db, DHS Quickstats Data for Sao Tome and Principe\n",
            "    Accessing data for resource 0f2ef8c4-353f-4af1-af97-9e48562ad5b1, Global WFP countries\n",
            "    Accessing data for resource 7c48c192-36d4-422e-996d-19cc62e2e4fd, Zimbabwe - Food Prices\n",
            "    Accessing data for resource 50c825b6-8fa4-47a7-8b23-e1d662c6ecc9, 3W_CAR_Dec2023\n",
            "    Accessing data for resource 99a1c9a3-98e0-427c-8e4b-e7c70889ba97, OCHA_CAR_COMPIL_PDI_REF_OCTOBRE_2022.xlsx\n",
            "Processing 20 of 612 (3.27%) resources\n",
            "    Accessing data for resource 757f7dcb-6c59-4595-a16b-d820f0c77a38, CERF Allocations (HXLated).csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 14:59:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=projectid\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: projectid\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 14:59:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=projectid\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: projectid\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 697c656a-684c-4ae1-9849-0db4d6b732ad, DTM Yemen Displacement Tracking  11 - 17 February 2024\n",
            "    Accessing data for resource 2ca8e86c-461c-482f-ac5e-9cdb8871a4ad, Lebanon - IDP Tracking Dataset - ROUND 24- 15-02-2024 - HDX.xlsx\n",
            "    Accessing data for resource c5ce40d6-07b1-4f36-955a-d6196436ff6b, EMDAT-country-profiles_2024_07_29.xlsx\n",
            "    Accessing data for resource a1a73231-7a96-4501-9675-97cddf3c3893, Suite of Food Security Indicators for Zimbabwe\n",
            "    Accessing data for resource 26e65696-95c3-4eac-8d18-c150526e34e0, Deflators data for Zimbabwe\n",
            "    Accessing data for resource b81e33f7-8942-4113-b675-8fbffc8a714f, Exchange rates data for Holy See\n",
            "    Accessing data for resource 6f659727-e3d8-4eef-bc2b-a3a836b65243, wpdx_water_points_hti\n",
            "Processing 30 of 612 (4.90%) resources\n",
            "    Accessing data for resource 6926dff7-658a-49e1-8d61-0ed8a983fbe1, ipc_global_national_long_latest.csv\n",
            "    Accessing data for resource a6f9f9b8-d265-4099-9ead-5e65fd26c074, Mozambique Crisis - MT Round 20 (Dec 2023).xlsx\n",
            "Dataset who-data-for-wallis-and-futuna-islands not found!\n",
            "Processing 40 of 612 (6.54%) resources\n",
            "    Accessing data for resource f87b7dd7-9d84-4e56-a334-402f5aec621e, ss_20230216_3w_op_january-december_2022.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:00:12 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=name, #reporting+or+implementing+org?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: name, #reporting+or+implementing+org?\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:00:12 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=name, #reporting+or+implementing+org?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: name, #reporting+or+implementing+org?\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource e77f3400-5d43-4a46-81a9-bf86e8958141, InterAction Member Data for Zimbabwe\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:00:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags='activtiy+working_group\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 'activtiy+working_group\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:00:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags='activtiy+working_group\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 'activtiy+working_group\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource e58625dc-946c-47aa-a086-28d30cf4fe58, Nouvelles admissions CRENAS, CRENI et CRENAM Sep 2023\n",
            "    Accessing data for resource 10a1749b-1484-496e-b505-b6a34a40ec4a, OCHA_SOM_Operational_Presence_3W_data_Oct2022.xlsx\n",
            "    Accessing data for resource d7c53285-fc22-4468-82fa-da7f087b60ca, 2023 Consolidated 3W data April to 31 Dec_hxl\n",
            "    Accessing data for resource 1bb1fff2-1a10-4024-ade6-d4a0dd499d8d, ECB_FX_USD-quote.csv\n",
            "    Accessing data for resource 15b313fb-9852-468f-a1eb-a83323338cd8, zimbabwe-healthsites-csv-with-hxl-tags\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:11 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:11 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource ca29f3a5-1e8e-4b57-bbb7-3e3469059845, zambia-healthsites-csv-with-hxl-tags\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp, part_time_beds\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp, part_time_beds\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp, part_time_beds\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: x, y, osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp, part_time_beds\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 50 of 612 (8.17%) resources\n",
            "    Accessing data for resource 08f68c8c-2e64-47e6-8566-9663e8c2da28, tokelau-healthsites-csv-with-hxl-tags\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:01:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: osm_id, osm_type, completeness, addr_housenumber, addr_street, addr_postcode, addr_city, changeset_id, changeset_version, changeset_timestamp\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 63390977-a035-4aa6-be1c-3e520b4cbb3e, Human Development Indicators for Zimbabwe\n",
            "    Accessing data for resource 62c167b6-2e99-494a-9cac-1630ef1b1369, Human Development Indicators for Somalia\n",
            "    Accessing data for resource 227b85c5-8e3b-4815-af04-bb0e66e183a7, yem_pin_2024.xlsx\n",
            "    Accessing data for resource dd12a3c8-b9e7-4b80-9835-f982dd0a4f76, afghanistan-natural-disaster-incidents-from-january-to-november-2023.xlsx\n",
            "    Accessing data for resource bc1f0808-636a-4aed-a4c1-7af540bd4d4f, Conflict Data for Zimbabwe\n",
            "    Accessing data for resource 3c4eb1db-3e72-4b85-a1d5-f0c7ffbf4582, DTM Chad Site and Village Assessment Round 21\n",
            "    Accessing data for resource d743b4b4-35d7-4e4d-80b9-18807b5df0b8, DTM Burundi_21_27_Janvier_2024 Emergency Event Tracking.xlsx\n",
            "Dataset mozambique-3ws-2023-jan-nov not found!\n",
            "    Accessing data for resource ebf41b1a-5b24-422d-be52-77e0c4c7a704, List of airports in Myanmar (HXL tags)\n",
            "Processing 60 of 612 (9.80%) resources\n",
            "    Accessing data for resource b9c66a29-6470-4954-8e4a-09309afe7855, hdx-hxl-datasets.csv\n",
            "    Accessing data for resource 2e130cdf-c850-4533-b2f3-e961adbec48a, Combined Indicators for Zimbabwe\n",
            "    Accessing data for resource 70f647af-d05b-4038-8c74-e925da8d333d, [Admin2] Data on displaced persons in UNRWA shelters by governorate\n",
            "    Accessing data for resource 33a3fe39-b88c-40bd-a213-fe5d8d5389ad, hti-pop-statistics.xlsx\n",
            "    Accessing data for resource 9239b72a-ad99-42c7-ae80-a22492435b80, UKR_3W_2021_Q3.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:06 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=adm1 +code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: adm1 +code\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:06 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=adm1 +code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: adm1 +code\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 19440fa1-7bc7-4cf7-a4c3-afaef150df52, DTM Haiti - Artibonite, Centre , Grande Anse, Nippes, South, South-East and West  (December 2023) - Baseline Assessment - Round 5.1\n",
            "    Accessing data for resource 49cb8fb5-bd40-4cc8-be58-330bc6c2585f, Uganda - Multi-Hazard Response/DRR Platform (December 2023)\n",
            "Error accessing data for resource 49cb8fb5-bd40-4cc8-be58-330bc6c2585f, Uganda - Multi-Hazard Response/DRR Platform (December 2023) ... Download of https://data.humdata.org/dataset/acc482ab-7ef8-4135-aa20-e75e110ebf9e/resource/49cb8fb5-bd40-4cc8-be58-330bc6c2585f/download/dec-2023-uganda-multihazard-dataset_public-hdx.xlsx failed in retrieval of stream!\n",
            "    Accessing data for resource 295cd9e4-8464-43ee-ad17-47196991a1f7, Demographics and locations of forcibly displaced and stateless persons (Global)\n",
            "Processing 70 of 612 (11.44%) resources\n",
            "    Accessing data for resource d3f24b62-82d6-4b12-97d1-66344615db94, Ethiopia Who is Doing What Where (4W) - December 2023.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=lat2, long\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: lat2, long\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=lat2, long\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: lat2, long\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource a10ae1ac-533f-4b0a-bc5a-d04bad3ef132, UKR 2024 HNRP PiN Severity Targets Activities 20240119.xlsx\n",
            "    Accessing data for resource e1953bbd-432e-4449-b417-dae7487663d0, DTM Cameroon Baseline Assessment Round 27\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 15\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 15\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource ec934f6d-ceca-4f34-acd2-5d567fd88a84, DTM Mali Baseline Assessment Round 78\n",
            "    Accessing data for resource 47902ccd-7d9d-4bfd-80b4-c3e4f5c38e20, Round 31 — Area Baseline Assessment (Raion level)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:45 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=adm2+name+eng\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: adm2+name+eng\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:45 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=adm2+name+eng\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: adm2+name+eng\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 193408bd-1016-47c4-9560-7015cd938aac, Afghanistan COVID-19 Stats\n",
            "    Accessing data for resource fae834e9-f3cc-42db-a964-7f4c127e5019, HTI_HNO_2024.xlsx\n",
            "    Accessing data for resource 0943c5b5-06b3-43ff-8574-6fc5aae7c459, access-classification-september-2023_hxl\n",
            "    Accessing data for resource ad7efeab-75f1-4039-a643-308cf86aefc5, CAR_Malnutrition\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:57 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+mam_under_6-59months, #indicator+mas_under_6-59months, #indicator+mcg_6-59 mois\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+mam_under_6-59months, #indicator+mas_under_6-59months, #indicator+mcg_6-59 mois\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:02:57 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+mam_under_6-59months, #indicator+mas_under_6-59months, #indicator+mcg_6-59 mois\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+mam_under_6-59months, #indicator+mas_under_6-59months, #indicator+mcg_6-59 mois\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 6157fadd-d04d-4747-88cd-77749628731b, COL_345W_Jan-Jun2021.xlsx\n",
            "Processing 80 of 612 (13.07%) resources\n",
            "    Accessing data for resource 16251491-9bc1-461a-9c45-110d3d3245ca, tcd_hpc2023_rev_pin-cible_20231006.xlsx\n",
            "    Accessing data for resource e9ce2fb4-3527-4576-8aec-8be688a18db0, nigeria-acute-malnutrition-sam-and-mam-oct-2023.xlsx\n",
            "    Accessing data for resource 09916f6a-d52a-46a7-91aa-35e7d593acf2, Myanmar_HPC_2024\n",
            "    Accessing data for resource 88eda02e-bdf0-4fa8-b914-f6113d22b206, TCD_VIZ_Nombre_Retournés_Niveau_Admin2_20231217.xlsx\n",
            "    Accessing data for resource 81a02eaf-f99f-43f4-b484-60e8494df81d, 2024_humanitarian_profile_10102023.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:03:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+host community\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+host community\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:03:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+host community\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+host community\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 9e82ad7e-470a-468a-a1fa-ab1bbc5ba80e, afg_hno_pin_2024.xlsx\n",
            "    Accessing data for resource c9836eb9-85be-4051-b638-f0a23c5a7ebb, south-sudan-humanitarian-needs-and-response-plan-2024_hxl.csv\n",
            "    Accessing data for resource 1a3206f3-2ac6-49e0-a70c-04206f651e92, mli-hno-2024\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:03:31 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+m+age>60, #inneed+f+age>60, #targeted+m+age>60, #targeted+f+age>60\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+m+age>60, #inneed+f+age>60, #targeted+m+age>60, #targeted+f+age>60\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:03:31 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+m+age>60, #inneed+f+age>60, #targeted+m+age>60, #targeted+f+age>60\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+m+age>60, #inneed+f+age>60, #targeted+m+age>60, #targeted+f+age>60\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 2c828f0f-565f-4f9d-b2ea-220126f49602, sdn_hnrp_2024_122123_hxl.csv\n",
            "    Accessing data for resource 605ddbac-37d7-47c1-84a4-a863a4023f83, UNHCR_UKR_dataset\n",
            "Processing 90 of 612 (14.71%) resources\n",
            "    Accessing data for resource ca1d748a-63a8-48d9-8503-e0a85516be44, cod_hno_2024\n",
            "    Accessing data for resource 793e66fe-4cdb-4076-b037-fb8c053239e2, global_pcodes.csv\n",
            "    Accessing data for resource 237e1704-29c9-4a8f-9f91-9e3dac624d7e, 3W_All_Clusters_March_2022\n",
            "    Accessing data for resource 45331aaa-e651-4a16-a8a0-92a51b3734c0, OxCGRT_CSV\n",
            "    Accessing data for resource 38807c10-84a7-4aec-8a00-a4e0e052a553, haiti-healthsites_hdx\n",
            "    Accessing data for resource 86f92eb8-504e-4685-bcc6-85121153c60e, DTM Somalia Baseline Assessment Round 2\n",
            "    Accessing data for resource b9718d0d-4c2f-48ee-a283-350b97e892c5, somalia-2023-post-gu-acute-malnutrition-burden-and-prevalence-by-district-21-sep-2023.xlsx\n",
            "    Accessing data for resource 796d90e7-23b6-499b-a16f-b20bc4cd47e6, DTM Ethiopia Site Assessment Round 34\n",
            "    Accessing data for resource f8e33345-28d7-4393-9708-afc7f3eca9aa, DTM Ethiopia Village Assessment Sep23 R17\n",
            "    Accessing data for resource 34c7a53a-20a7-433f-8876-d71b16e73e28, ner_movements-population_30 novembre 2023\n",
            "Processing 100 of 612 (16.34%) resources\n",
            "    Accessing data for resource 138a3a3e-11ce-42e1-8649-dc656eb10f43, Synthese_besoins_reponse_gaps_Dec2023\n",
            "    Accessing data for resource 2dc2e84f-7d85-4549-b7a0-12f00b0a268c, car_hno_2024\n",
            "    Accessing data for resource 91346a26-1109-4f45-940e-1b43c38059bc, nigeria-floods-2023.xlsx\n",
            "    Accessing data for resource e1129533-f1b8-4706-a79e-778ebaeedc88, VEN_5W_Jan-Oct2023.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:09:17 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=sal\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: sal\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:09:17 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=sal\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: sal\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Error accessing data for resource e1129533-f1b8-4706-a79e-778ebaeedc88, VEN_5W_Jan-Oct2023.xlsx ... list index out of range\n",
            "    Accessing data for resource d394406d-7d81-4821-a4bc-cae55119daab, 3W_TCD_Nov2023\n",
            "    Accessing data for resource 09cef61d-b5fc-42c6-9cf5-99371e938b40, BFA_Innindation_2023\n",
            "    Accessing data for resource 79abb4c0-ef91-43c0-bc2c-7a54179a6eb2, YEM_4W_Jan-Sept-2023.xlsx\n",
            "    Accessing data for resource bfc5a2d0-23e7-46e7-bde4-1a3a54f45d8e, BFA_HNO_2024\n",
            "Dataset mozambique_-humanitarian-needs not found!\n",
            "Processing 110 of 612 (17.97%) resources\n",
            "    Accessing data for resource 2114c854-d95c-4974-9ff0-2d59cd1d49fa, 2022 IRN SHCC Health Care Data.xlsx\n",
            "    Accessing data for resource 28be64d3-adaf-4f61-887c-87a8b5d9c625, GHO 2024 Section 3.xlsx\n",
            "    Accessing data for resource 501b929d-012b-4557-b5bf-2003fd4e3661, DTM Kenya — BA/MSLA - Samburu County — Round 1\n",
            "Error accessing data for resource 501b929d-012b-4557-b5bf-2003fd4e3661, DTM Kenya — BA/MSLA - Samburu County — Round 1 ... Download of https://data.humdata.org/dataset/db3e1a76-76d8-4206-9e5e-382336c51472/resource/501b929d-012b-4557-b5bf-2003fd4e3661/download/dtm_kenya_ba_msla_round_1_samburu_county_jan_2023_ext.xlsx failed in retrieval of stream!\n",
            "    Accessing data for resource aa0efd37-f743-4f47-8570-dff08735218f, MALI_3W_September_2023\n",
            "    Accessing data for resource 8f39a8df-555a-4ceb-9983-82d286e232ab, Sudan Schools (HXLated)\n",
            "    Accessing data for resource 5f8631be-096e-4629-9c90-49d8bbcfa874, List of Schools in Ethiopia (HXLated)\n",
            "    Accessing data for resource 29c2a3cf-2ad3-4841-b41d-b1c82c8be7c2, NER_SITUATION DES INONDATIONS AU 09 OCTOBRE 2023 DSHU MAHGC.xlsx\n",
            "    Accessing data for resource 0defb6d6-b47b-465e-ba11-f70545e1efa7, Key figures (HXLated)\n",
            "    Accessing data for resource 88990abe-db22-41a7-8d22-93129eb19286, who-is-doing-what-and-where_nga_3w_jul_sept_2023.xlsx\n",
            "    Accessing data for resource 0f2b920b-7985-48cf-8fab-872bc0403f80, afghanistan-3w-july-september-2023.xlsx\n",
            "Processing 120 of 612 (19.61%) resources\n",
            "    Accessing data for resource cf0e892b-4565-4986-a2a5-a64d37810aca, Countries and Territories - CSV\n",
            "    Accessing data for resource a3e9bbbb-7bda-4328-b17e-ab2955450837, unocha-glide-numbers.csv\n",
            "    Accessing data for resource 9a7aa9f3-74f1-4d1c-b36c-536025eb7f57, syria-3w-jan-to-mar-2023.xlsx\n",
            "    Accessing data for resource 25e11652-c50b-485f-b093-4515441a57f2, Resultados PIN intersectorial y sectorial por Municipio - PiN 2023 Colombia.xlsx\n",
            "    Accessing data for resource dea66c96-8a2c-40d5-af6d-8b8771038ce0, cmr_hno_hrp_2023\n",
            "    Accessing data for resource 777d6429-8fbd-431c-9402-dc70b8603251, Situation des PDI_31_mars 2023\n",
            "    Accessing data for resource 362cd02c-c2e3-47f6-87e2-d990981ab126, DRC_Climate impact April 2024\n",
            "Error accessing data for resource 362cd02c-c2e3-47f6-87e2-d990981ab126, DRC_Climate impact April 2024 ... <HXLIOException: Cannot find CSV file or Excel content in zip archive>\n",
            "    Accessing data for resource 6ca82f30-a4a4-4345-b49e-c470836f5859, DTM South Sudan Flow Monitoring Nov22 to Oct23\n",
            "    Accessing data for resource fa593f22-8649-4393-80a8-4598af687e68, Attacks on Education 2018–19.xlsx\n",
            "Error accessing data for resource fa593f22-8649-4393-80a8-4598af687e68, Attacks on Education 2018–19.xlsx ... list index out of range\n",
            "    Accessing data for resource 0236aef7-804d-41dd-b4aa-9994bbfc9e11, cmr_3w_national_sep_2023\n",
            "Processing 130 of 612 (21.24%) resources\n",
            "    Accessing data for resource 1f222b61-614d-43f3-b20b-4fcdd44c87fd, DRC-nutrition-data Mai 2023\n",
            "    Accessing data for resource 75ac140e-9b52-4bbd-96fc-5eb52d0e76a2, 3W Burkina Faso July-August 2023\n",
            "    Accessing data for resource fa72f791-d515-4be7-b355-dbb0cfa3572f, DRC - Baseline Assessement - Countrywide Monitoring - Round 1.xlsx- Copy.xlsx\n",
            "    Accessing data for resource acdbbe58-94e6-4abd-af13-e10efdb15636, somalia-drought-affected-targeted-reached-by-location\n",
            "    Accessing data for resource 60bd6057-4df7-462a-9712-900c3d75d39e, DRC – Mobility Tracking Atlas – Round 3 – October 2023\n",
            "    Accessing data for resource 29ba852c-aca2-40b2-b48e-7cfb74c7219f, DTM Libya Migrants Baseline Assessment Round 48\n",
            "    Accessing data for resource 7c6b0025-ee35-4b81-8724-c66163bd887c, statistique_rapatriement_Septembre-2017_Octobre-2023\n",
            "    Accessing data for resource e5eb34af-1dd1-4a5d-b72b-8246df899c9d, Pakistan – Community Needs Identification (Flood Response) – Round 3\n",
            "Processing 140 of 612 (22.88%) resources\n",
            "    Accessing data for resource f057e27e-51e1-4a35-9d30-37f89b380459, Asylum applications by asylum seekers originating from Cook Islands\n",
            "    Accessing data for resource 78227d5f-3fa4-4404-8122-a6a7267ff1d2, Updated PINd-in-the-provinces-nk-sk-and-ituri June-2023\n",
            "    Accessing data for resource c83c06ce-a9b4-4da4-95b8-bdf90a45a2be, DTM DRC - BA North Kivu October 2023 - R10.xlsx\n",
            "    Accessing data for resource ed6827b6-fa77-4d9c-9129-2ab98377b016, Immunization Campaigns Impacted\n",
            "    Accessing data for resource 3ef4e542-614e-4873-95fa-375d4f52a56c, DTM DRC BA South Kivu Oct 2023 R11\n",
            "    Accessing data for resource e920e3be-44c0-4744-8199-4a5c32e76ae9, DTM DRC - BA Tanganyika October 2023 - R16\n",
            "    Accessing data for resource 19d77181-f15d-4a36-b490-79b14f87c560, NER_Carte_sanitaire_du_Niger.xlsx\n",
            "    Accessing data for resource 7680329f-7abf-462d-b349-61ffd81f1dac, NER_ Liste_Etablissements du Niger.xlsx\n",
            "    Accessing data for resource b6f75de1-4b73-4870-9c30-0e3d1c919e76, Round 14 IDP Estimation 25 September 2023\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:18:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=* estimates in donetska, zaporizka, luhanska and khersonska oblasts (blue text) are likely under-represented due to limited coverage of government-controlled areas only, as well as the limited number of respondents reached through the random digit dial. the estimation for luhanska is taken into account only within the total population estimation.\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: * estimates in donetska, zaporizka, luhanska and khersonska oblasts (blue text) are likely under-represented due to limited coverage of government-controlled areas only, as well as the limited number of respondents reached through the random digit dial. the estimation for luhanska is taken into account only within the total population estimation.\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:18:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=* estimates in donetska, zaporizka, luhanska and khersonska oblasts (blue text) are likely under-represented due to limited coverage of government-controlled areas only, as well as the limited number of respondents reached through the random digit dial. the estimation for luhanska is taken into account only within the total population estimation.\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: * estimates in donetska, zaporizka, luhanska and khersonska oblasts (blue text) are likely under-represented due to limited coverage of government-controlled areas only, as well as the limited number of respondents reached through the random digit dial. the estimation for luhanska is taken into account only within the total population estimation.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 9026e57c-ad7b-4786-8841-b36264097f6c, 3W_Haiti_HDX_20230511\n",
            "Processing 150 of 612 (24.51%) resources\n",
            "    Accessing data for resource 9d396e61-e708-48ab-b51e-f50579fa169a, BDI_HNO_2023\n",
            "    Accessing data for resource 6c15e318-635a-4de8-9b8c-58c7c4e8af97, DTM DRC BA Ituri Aug2023 R10\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:18:50 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+ind+idp+\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+ind+idp+\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:18:50 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+ind+idp+\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+ind+idp+\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 668dbc8d-442a-4a12-96d2-0682313b586b, Round 14 — DTM Ukraine Returnees Dataset\n",
            "    Accessing data for resource c50a1711-f5f8-471b-9887-85d396f19ddd, DTM Burundi Baseline Assessment Round 73\n",
            "    Accessing data for resource 53c40477-6596-4209-b6a8-d587aac0dfe8, suivi-du-bilan_2022_nutrition\n",
            "    Accessing data for resource 6341ffbf-85a3-44ec-b3eb-7ff300174252, DTM South Sudan Baseline Assessment Round 14\n",
            "    Accessing data for resource 5fa55cb0-326a-4629-a056-eaf37aeff9d9, DTM Libya Baseline Assessment Round 45\n",
            "    Accessing data for resource c524f6bd-f023-4cf3-a52d-ed0e9be5242b, hno-nigeria-2023_hxl-tags.xlsx\n",
            "    Accessing data for resource 25dca590-f12c-40e1-817a-9da1344bca5e, DTM Mozambique Site Assessment Round 22\n",
            "    Accessing data for resource b1e2a1f7-bd73-42a7-9cde-6227a78d4b77, DTM Madagascar Baseline Assessment Round 5\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:19:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0, 0, 0\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0, 0, 0\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:19:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0, 0, 0\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0, 0, 0\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 160 of 612 (26.14%) resources\n",
            "    Accessing data for resource 62a5dbd5-ecd5-476e-8cfc-6225331e1377, DTM DRC BA Kasai Central Dec2020 R7\n",
            "    Accessing data for resource 46e11b83-f60a-4796-a027-e46483193fa6, DTM Lebanon — Migrant Presence Monitoring — Round 3\n",
            "    Accessing data for resource 175fe47a-ab68-495d-8ac0-defce8e4924d, TCD_DATA_SMART2021\n",
            "Error accessing data for resource 175fe47a-ab68-495d-8ac0-defce8e4924d, TCD_DATA_SMART2021 ... list index out of range\n",
            "    Accessing data for resource ce85c4d9-3cea-4e59-833a-ff19e91b9006, NER_Sep_2023\n",
            "    Accessing data for resource 3bc1cf5a-a46d-47a5-8f7f-93789f5db5b9, Health Facilities\n",
            "    Accessing data for resource 0b8dd14b-a9b9-408f-8e9d-08e3dbc84ef7, somalia-drought-key-figures\n",
            "    Accessing data for resource e1d706aa-f2cc-43b0-80f1-60280945371c, DTM CAR Baseline Assessment Round 19\n",
            "    Accessing data for resource 2f20951f-aaf8-4c69-a3be-7ad116ca3569, Black Sea Initiative Vessel Movements\n",
            "    Accessing data for resource 79c65240-ebc9-4f04-a7d1-5cd4db860f5b, DTM Sudan Migrants Master List Round 6\n",
            "Processing 170 of 612 (27.78%) resources\n",
            "    Accessing data for resource bf39bd1e-c935-42a0-9a48-a86313416ef3, DTM Sudan Returnees Master List Round 6\n",
            "    Accessing data for resource b27deb4b-99c8-4104-b6e2-9263b9b47016, DTM Sudan IDPs Master List Round 5\n",
            "    Accessing data for resource a4ffbd9f-46f6-4733-aea4-54bee4631b45, Internal displacements (new displacements) associated with disasters\n",
            "    Accessing data for resource 2ec3bca9-0d3d-4914-ba94-f50418440181, Internal Displacements (New Displacements) – IDPs\n",
            "    Accessing data for resource ebf434c1-56e9-49c6-baff-4b9ab1a67cec, DTM Mongolia — Rapid Assessment - Migration, Population and Needs - Round 3\n",
            "    Accessing data for resource 8fba4760-847d-4c73-b6a4-ffdbfb4bf7ac, eth_hrp_2023_qc.csv\n",
            "    Accessing data for resource cbe79019-9c98-498c-bf2a-ea720cb4ebdd, DTM Burundi Multi Sectoral Location Assessment Round 3\n",
            "    Accessing data for resource e5ea2d62-3586-48d1-94cb-268983e14cb7, Covid-19 confirmed cases by state\n",
            "    Accessing data for resource d45644e8-394c-4fe6-a174-36739c5e36a4, Covid-19 Data Explorer - national data\n",
            "    Accessing data for resource 878e400c-17ad-4806-b755-b1aa983ad8a2, DTM Ethiopia - VAS - Tigray - R16\n",
            "Processing 180 of 612 (29.41%) resources\n",
            "    Accessing data for resource ba90e7e1-d5de-4093-9f88-7235267b6b06, Somalia Access severity- 2023.xlsx\n",
            "    Accessing data for resource 1209b98e-ad70-4d86-ae7e-96c8b426bf6f, Mali_suivi des inondations\n",
            "    Accessing data for resource 8a741b70-b122-4a83-9b70-0f8195bc95e2, kenya-drought-affected-targeted-reached-by-location\n",
            "    Accessing data for resource 17755693-2662-4642-b36f-9dda640eb277, Food basket change\n",
            "    Accessing data for resource a43c1bb3-084a-422e-aa9c-16c310b76209, Ethiopia-2021-humanitairan-needs-overview\n",
            "    Accessing data for resource 22935fd3-2639-42e5-ba42-7fea35537f08, DTM Ethiopia - SA - Tigray - R33\n",
            "    Accessing data for resource cf325a25-1994-4083-a209-4f4c7e107f9f, AWSD_Verified_security_incidents_2023-08-10.csv\n",
            "    Accessing data for resource f23d8f2f-76a1-4c61-8b72-4e101ddd4c30, Kazakhstan - International Migrant Workers Survey - Round 2\n",
            "    Accessing data for resource e5f334a8-23a6-4e4c-b08b-a48f3ffb3787, DTM Nigeria Baseline Assessment Round 43\n",
            "    Accessing data for resource 27469ccb-7c9e-4a48-ad3d-b692d6ae3ce4, ethiopia-drought-affected-targeted-reached-by-location\n",
            "Processing 190 of 612 (31.05%) resources\n",
            "Dataset cash-based-programming-in-somalia-2018 not found!\n",
            "    Accessing data for resource b43ffad8-3684-4de0-bc51-9d15b111d4d2, HoA Drought Related Data - Kenya Drought Related IDPs.csv\n",
            "    Accessing data for resource 43e6353e-2329-421b-b122-d6e52bf77f91, DTM Nigeria North Central & West Site Assessment R11.xlsx\n",
            "    Accessing data for resource caafa66a-58ed-4312-91e0-dacccb1e1a51, DTM Nigeria Site Assessment Round 43\n",
            "    Accessing data for resource dc7a5656-d557-404f-8b1d-494c7bbd0112, grd_admgz_PS.xlsx\n",
            "Error accessing data for resource dc7a5656-d557-404f-8b1d-494c7bbd0112, grd_admgz_PS.xlsx ... <HXLTagsNotFoundException: HXL tags not found in first 25 rows>\n",
            "    Accessing data for resource 798d2e35-951e-4313-832f-16c853b616ed, YEM_Pop_2021.xlsx\n",
            "    Accessing data for resource d1de55cf-4d64-4444-a9e4-f3a89ed31a3b, transactions.csv\n",
            "    Accessing data for resource 36208638-b15a-4318-8056-994f4a67232a, GHO-mid-year-update-2023.xlsx\n",
            "    Accessing data for resource e7c868d3-9cb3-473f-895d-0efbb4a83c7f, Drought Induced Displacement\n",
            "    Accessing data for resource 6ec447fa-47b9-4e6c-a522-0ff20271591a, Haiti Floods - 2023.xlsx\n",
            "Processing 200 of 612 (32.68%) resources\n",
            "    Accessing data for resource c387ce97-8f82-4a0d-9cb0-e1009f14a896, DTM DRC Site Assessment North Kivu Jul2019 R37\n",
            "    Accessing data for resource 39d85341-4914-4c89-86e5-f8ae88b5817b, SYR_Access_Severity_HCT_May_2023.xlsx\n",
            "    Accessing data for resource b9e438e0-b68a-49f9-b9a9-68c0f3e93603, DTM Afghanistan Settlements Round 16 Baseline Assessment\n",
            "    Accessing data for resource 6f0eb2b4-45db-4873-8c7c-6ba5dbcd41e0, HTI_20230601_Cartographie des contraintes d'acces.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:24:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#zone-specifique, #zone-specifique + code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #zone-specifique, #zone-specifique + code\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:24:19 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#zone-specifique, #zone-specifique + code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #zone-specifique, #zone-specifique + code\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 6ad4b44b-d9d4-4832-b39a-da7f1c912930, 3W_BDI_Mar2022\n",
            "    Accessing data for resource 121d6cbe-92ff-4059-841f-26d8a8887ac9, global-3w-2023-06-08.xlsx\n",
            "    Accessing data for resource 7106abfe-384a-4e26-86ab-eed828e91d63, 2022 SHCC Incident Data.xlsx\n",
            "    Accessing data for resource 12fc93c2-39f6-4b8b-8f18-2a135e4d1a45, Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence\n",
            "    Accessing data for resource cf461e2e-4ae2-439d-a6c6-ec0c5e2f5bad, Iraq populated places 2021 P-coded.xlsx\n",
            "    Accessing data for resource 0130fc33-dd0a-4547-bf13-1e1514e4c134, DTM South Sudan Event Tracking - Jan-Dec 2022\n",
            "Processing 210 of 612 (34.31%) resources\n",
            "    Accessing data for resource b08f71f9-ba7e-4295-ba15-035a72f32fe6, 2016-2023 Attacks on Vaccination Campaigns.xlsx\n",
            "    Accessing data for resource 8fd2b30c-6030-4daa-886c-0e3a6016025c, CAR_Cluster Education BD des écoles_08052023.xlsx\n",
            "    Accessing data for resource 6332355d-ec41-4136-bb12-42bbb8bc0c28, DTM Malawi Site Assessment - Cyclone Freddy - Round 1\n",
            "    Accessing data for resource 84aba8ee-67ec-4c95-9769-e9ffff44ca7c, IFRC Appeals Data for Uzbekistan\n",
            "    Accessing data for resource fa91d147-873c-44d3-b890-554f4eb407e7, BD_mouvements populations_ope_06052024.xlsx\n",
            "Error accessing data for resource fa91d147-873c-44d3-b890-554f4eb407e7, BD_mouvements populations_ope_06052024.xlsx ... <HXLTagsNotFoundException: HXL tags not found in first 25 rows>\n",
            "    Accessing data for resource ab129f6a-fea2-4809-95d6-373015159970, PSE - Who What Where(3w)\n",
            "    Accessing data for resource cb334cde-fe6b-4ac5-b8a1-b09dfa76cc07, displacement_data\n",
            "    Accessing data for resource 6b7b5f02-9379-4aae-a0b0-03b969d33799, displacement_data\n",
            "    Accessing data for resource d3941c15-705f-4dd3-a552-961b877a57a5, Malawi Humanitarian Organizations Response TS Freddy (300324).xlsx\n",
            "    Accessing data for resource ba4ef454-1ebb-46e2-9fef-49baa40fe01f, Interagency Response Plans in 2023\n",
            "Processing 220 of 612 (35.95%) resources\n",
            "    Accessing data for resource 4b6d680d-e4c4-4900-8a0e-15fa2dafdb31, afghanistan-3w-october-to-december-2022.xlsx\n",
            "    Accessing data for resource ae7a53d1-7e09-4eaa-9790-13ddcb140e62, help_requests_twitter.xlsx\n",
            "    Accessing data for resource 8bf028c0-65e8-4e6c-9114-32293957d3b9, Data HXL.csv\n",
            "    Accessing data for resource 1df585b2-3a1d-4781-a4b4-a879bb4f2598, Data.hxl.csv\n",
            "    Accessing data for resource e4da3c36-204c-476c-a666-a88afbd23633, 2014-2018 - Somalia Death Rates.xlsx\n",
            "    Accessing data for resource d5ef53ae-a323-41c4-8f74-c719529cb2e4, Turkiye & Syria Earthquakes - OCHA SitReps\n",
            "    Accessing data for resource 6683462f-232f-4c81-b6c1-ea2bc0a3c5a7, iraq_3w_jan-to-dec_2022_longtable_20230124.xlsx\n",
            "    Accessing data for resource 3d1f5dc4-858d-4ee4-a078-d3e1c10f4276, pk_lang_V01_MICS.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:01 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_bal, #indicator+lang+pct+iso639 3_brh, #indicator+lang+pct+iso639 3_prs, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_hnd, #indicator+lang+pct+iso639 3_khw, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_pan, #indicator+lang+pct+iso639 3_pus, #indicator+lang+pct+iso639 3_skr, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_snd, #indicator+lang+pct+iso639 3_urd\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_bal, #indicator+lang+pct+iso639 3_brh, #indicator+lang+pct+iso639 3_prs, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_hnd, #indicator+lang+pct+iso639 3_khw, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_pan, #indicator+lang+pct+iso639 3_pus, #indicator+lang+pct+iso639 3_skr, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_snd, #indicator+lang+pct+iso639 3_urd\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:01 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_bal, #indicator+lang+pct+iso639 3_brh, #indicator+lang+pct+iso639 3_prs, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_hnd, #indicator+lang+pct+iso639 3_khw, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_pan, #indicator+lang+pct+iso639 3_pus, #indicator+lang+pct+iso639 3_skr, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_snd, #indicator+lang+pct+iso639 3_urd\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_bal, #indicator+lang+pct+iso639 3_brh, #indicator+lang+pct+iso639 3_prs, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_hnd, #indicator+lang+pct+iso639 3_khw, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_pan, #indicator+lang+pct+iso639 3_pus, #indicator+lang+pct+iso639 3_skr, #indicator+lang+pct+iso639 3_plk, #indicator+lang+pct+iso639 3_snd, #indicator+lang+pct+iso639 3_urd\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 00084aa2-52c6-4332-8077-9fb4955d7820, PiN_VBG_2023_hdx.xlsx\n",
            "Processing 230 of 612 (37.58%) resources\n",
            "    Accessing data for resource a14afbc7-2b59-42e9-8642-abd9a7169ceb, afghanistan_conflict_displacements_2022.xlsx\n",
            "    Accessing data for resource 31366c37-4803-4886-a608-768cd948cf7f, Palestine COVID-19 Cases by Governorate\n",
            "    Accessing data for resource 57e50360-b021-4772-9bc5-542e0af00c2a, herams_nampula-db_qtr-2-2022_hxl.xlsx\n",
            "    Accessing data for resource 235f892e-beea-4d90-8af3-19baf12ea5b9, hti_polbndl_rd_cnigs.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:47 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=# names\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: # names\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:47 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=# names\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: # names\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 92827be4-f6b3-4453-80cc-4c5c2e5d3d5f, DF_SITREP_COVID19.csv\n",
            "    Accessing data for resource f8e6e090-fa9a-41df-b561-267d8728af38, Desnutrición Aguda_HDX_2022.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:52 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:52 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource c17c4113-b885-4799-b24a-44562eb57947, PiN_Seguridad Alimentaria y Nutrición_2023.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:55 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:55 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 77b38c47-d802-4ae5-8445-73cd3132be22, pin_severity_hno_2023.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:59 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#severity + code, #severity +code +protection_mine-action, #inneed +ind +protection_mine-action\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #severity + code, #severity +code +protection_mine-action, #inneed +ind +protection_mine-action\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:26:59 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#severity + code, #severity +code +protection_mine-action, #inneed +ind +protection_mine-action\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #severity + code, #severity +code +protection_mine-action, #inneed +ind +protection_mine-action\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource cfe49c66-8975-44e4-b8e0-594b5ea2c5ee, SYR_FSA_PIN_2023.xlsx\n",
            "    Accessing data for resource 685d0768-17c5-4947-969d-4de446750704, SDG 4 Global and Thematic data\n",
            "Processing 240 of 612 (39.22%) resources\n",
            "    Accessing data for resource 1a856318-0c7f-4b9f-bc29-0c98a717b125, COVID-19 pandemic Attacks on Health Care in 2020.xlsx\n",
            "    Accessing data for resource dedff8c3-065b-413d-9d01-2570e82d566b, flood-response-monitoring-matrix-august-2022-coded-woredas.xlsx\n",
            "    Accessing data for resource 60ec36f5-59d9-40b6-983d-195b2e660276, CMR_EXNO_Data_InondationLC_MD_MT_V1.0_20221206.xlsx\n",
            "    Accessing data for resource 918b0b51-30d3-4d46-b0c5-1688bdf27b8e, SSD_floods_301122.xlsx\n",
            "    Accessing data for resource 63143067-46e0-4fb5-b131-d83ee45122ab, ETH_CapitalTowns 2021\n",
            "    Accessing data for resource 0061b888-67c4-4798-83f5-ea44df8587e4, who-is-doing-what-and-where_nga_3w_jul_sept_2022.xlsx\n",
            "    Accessing data for resource 65564e72-3052-4ed0-98cf-310896f7dbeb, ner_hno_pin_2023.xlsx\n",
            "    Accessing data for resource fce5688c-c019-4738-bf8e-4d1976fa4386, AFG-districts-affected-by-winter\n",
            "    Accessing data for resource 4e6d9fcf-67ea-4809-b279-fb55a3598b2c, 3W as of September 2022.xlsx\n",
            "    Accessing data for resource 146b9a31-ecc4-494c-9832-5788c4645fd8, COVID-19 - Vaccinations\n",
            "Processing 250 of 612 (40.85%) resources\n",
            "    Accessing data for resource 0247ce98-26d6-4d87-96b5-e977138fac47, DTM Madagascar - BA Drought in Gand Sud - Sep 2022\n",
            "    Accessing data for resource 84656a66-ea1c-4471-ae2b-58d9817d78cd, afghanistan-population-estimates-urban-rural\n",
            "    Accessing data for resource a4eef93c-95ef-478d-b35d-fa1756c8f839, FieldsData_3W_PHL_Aklan\n",
            "Dataset cameroon-displacement-idps-msna-iom-dtm not found!\n",
            "    Accessing data for resource ba3d2c05-d74c-40d3-95b9-864f2ed0dade, Retours PDI_au phase 2_Diffa_26082022.xlsx\n",
            "    Accessing data for resource 8f183884-0fb0-454e-9937-588a23e700b7, 3W_Regional_West_and_Central_Africa_Jun2022\n",
            "    Accessing data for resource bc978a1f-3527-48e6-8b84-41e9a0dd6d5f, Round_42_IDP_Dataset_September_2022.xlsx\n",
            "    Accessing data for resource 67185441-8eac-4ccb-a2c9-cceb18bdfb67, 20220706_airfields_hdx.xlsx\n",
            "    Accessing data for resource 2318ec9d-a553-485b-aaa6-70479285347a, afghanistan-3w-operational-presence-july-to-september-2022.csv\n",
            "    Accessing data for resource e324caa4-9152-4cc3-9cd7-228335e06632, South Africa – Baseline Ward Assessment – Round 1\n",
            "Processing 260 of 612 (42.48%) resources\n",
            "    Accessing data for resource a22a5c1a-5282-4ece-be0b-bf2dcf60d207, somalia-cumulative_drought_response_jan-june.xlsx\n",
            "    Accessing data for resource 45d048eb-71db-41f9-acbf-5672cd43d9af, Original Google Sheet\n",
            "    Accessing data for resource d9172671-88a8-488b-9872-bdc44c6a982f, Base_vacantes_laborales_demandadas_ofertas_laborales_migrantes_Enero_Junio.xlsx\n",
            "    Accessing data for resource 87bae300-8b66-452a-8da1-439664757ef2, somalia-population-statistics 2014\n",
            "    Accessing data for resource dea86131-37fd-40a7-be24-c01a4cacde11, 2022-HRP-Sectors-Response-Jan-Jun.xlsx\n",
            "    Accessing data for resource ac9730be-c858-4329-824d-be6cf4757a4d, IATI activities in Nepal\n",
            "    Accessing data for resource 7bc08fda-8a54-445a-ac48-d142a938d10f, SO_lang_admin0.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:30:21 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_arb, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_ymm, #indicator+lang+pct+iso639 3_xma, #indicator+lang+pct+iso639 3_som, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_arb, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_ymm, #indicator+lang+pct+iso639 3_xma, #indicator+lang+pct+iso639 3_som, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:30:21 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_arb, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_ymm, #indicator+lang+pct+iso639 3_xma, #indicator+lang+pct+iso639 3_som, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_arb, #indicator+lang+pct+iso639 3_unknown, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_swh, #indicator+lang+pct+iso639 3_ymm, #indicator+lang+pct+iso639 3_xma, #indicator+lang+pct+iso639 3_som, #indicator+lang+pct+iso639 3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 363cfc0e-74ce-4268-815c-8705a72ccd0d, afghanistan-3w-april-to-june-2022.xlsx\n",
            "    Accessing data for resource dfa9bde5-9f6f-43fd-af4d-b43bd8d0e124, Ukraine Flash Appeal - March to December 2022 PIN_HDX.xlsx\n",
            "    Accessing data for resource 25bcd859-16ff-449b-9b8e-707a724f5152, who-is-doing-what-and-where_nga_3w_apr_jun_2022.xlsx\n",
            "Processing 270 of 612 (44.12%) resources\n",
            "    Accessing data for resource 60d15653-37e7-433c-87a4-20826ef74456, HT_Climato-Hydro-Meteo_EMDAT_Data 20220623\n",
            "    Accessing data for resource 8161df6c-77ef-4627-9f17-1f81e975923b, HT_Acces a l'eau par commune 2022\n",
            "    Accessing data for resource a3634ad3-5375-4644-88b2-93693f1a7ab4, 220702_3W Typhoon Rai_Odette Consolidated HDX.xlsx\n",
            "Error accessing data for resource a3634ad3-5375-4644-88b2-93693f1a7ab4, 220702_3W Typhoon Rai_Odette Consolidated HDX.xlsx ... list index out of range\n",
            "    Accessing data for resource fc41924b-9ace-4a71-9a6a-031e24bf841c, 2021-HRP-Sectors-Response-Jan-Dec.xlsx\n",
            "    Accessing data for resource 9ef63381-65e4-4109-ac3f-10fc15964c6d, 2022-Hazards-planning-figures.xlsx\n",
            "    Accessing data for resource 78b2d175-cacf-4fc9-b6b0-9b7c02b002b5, homicidios_mujeres_feminicidios_EneroMarzo_2022.csv\n",
            "    Accessing data for resource f9545c69-0693-49f2-9c10-a0cb505648ff, COVID-19 health facilities in Afghanistan by Province.xlsx\n",
            "    Accessing data for resource 410c2a68-ee49-408f-b85e-f1c619f8eec2, Ethiopia_Confirmed COVID_19 Cases_fromMar13\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:31:11 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+infected+new+24hrs, #affected+infected+test+24hrs\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+infected+new+24hrs, #affected+infected+test+24hrs\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:31:11 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+infected+new+24hrs, #affected+infected+test+24hrs\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+infected+new+24hrs, #affected+infected+test+24hrs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 8cad612d-0245-45ae-ad25-1e9c9d82eb8b, Somalia COVID-19 cases by location\n",
            "    Accessing data for resource 18797eb3-7352-4f99-b117-ffcecbaeb3d9, Libya COVID-19 Cases by Location\n",
            "Processing 280 of 612 (45.75%) resources\n",
            "    Accessing data for resource 180183e8-d5fc-4e81-a8c3-29f80be2af6b, DTM Nigeria North Central & West Location Assessment R9\n",
            "    Accessing data for resource cb439fdf-23b4-4116-9ff2-a28d6c341e0f, DTM South Sudan Site Assessment Round 11\n",
            "    Accessing data for resource 615848f2-4f81-46ef-9cb8-aa50b11147e9, Haiti: Coronavirus (COVID-19) Subnational Cases\n",
            "    Accessing data for resource 8e15cffe-2815-4cbb-acd3-9a315bbc4d84, Child_protection_area_of_responsibility_organizations_hxl.xlsx\n",
            "    Accessing data for resource c9ea0e06-6357-4356-a8b6-acef2367d35e, DTM Nigeria North Central & West Baseline Assessment R9\n",
            "    Accessing data for resource c68af038-870a-4ee9-82bf-3413b65b2b55, Deflators data for Turkey\n",
            "    Accessing data for resource 329d810b-ca39-4bed-8f1b-e390ca4e210a, Conflict Data for Turkey\n",
            "    Accessing data for resource 9e839677-3ff0-44b3-992c-1a99e68df515, DOH-Epi Dengue Data 2016-2021.csv\n",
            "    Accessing data for resource 64392331-ff6c-44eb-b21d-5a21e8b2854f, Schools dataset cleaned and reviewed_hxl\n",
            "    Accessing data for resource f1f6b051-e63b-4c99-94fa-e2ac23dc5a56, DTM Uganda Baseline Assessment Round 1\n",
            "Processing 290 of 612 (47.39%) resources\n",
            "    Accessing data for resource d14adc69-7ce2-43b1-91f5-c0ea2e973055, victimas_hecho_dpto_2017_2021.xlsx\n",
            "    Accessing data for resource 7adad93b-4dd7-44db-938a-dbe4fcf49080, Affected Learners\n",
            "    Accessing data for resource ca984a7a-d4a3-4e33-aaab-b4055627db90, IRAQ 2022 HNO HRP Population PIN Severity Targets Activities Requirements.xlsx\n",
            "    Accessing data for resource 3cdb4783-cb99-4266-8f92-2df33c3ec570, DTM Zimbabwe Cyclone IDAI Baseline Assessment Round 7\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:30 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=affected+migrants+hh, affected+migrants+ind\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: affected+migrants+hh, affected+migrants+ind\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:30 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=affected+migrants+hh, affected+migrants+ind\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: affected+migrants+hh, affected+migrants+ind\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 5e42b51e-8395-40ec-90dc-fa4d20d90307, DTM Zimbabwe Village Assessment — Matabeleland South and Masvingo Provinces (November 2021)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:36 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=affected+migrants+hh\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: affected+migrants+hh\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:36 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=affected+migrants+hh\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: affected+migrants+hh\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource d2f9cbaf-5fb2-454d-a59c-f9e1849f21b1, NGA_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource da6cba36-f81f-4bf7-9ad3-b04cf52bedf9, BFA_Subnational_Covid19_HXL_HERA.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+healthworkers+1dose, #affected+cumulative+vaccinated+healthworkers+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses, #affected+vaccinated+healthworkers+2doses, #affected+cumulative+vaccinated+healthworkers+2doses\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+healthworkers+1dose, #affected+cumulative+vaccinated+healthworkers+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses, #affected+vaccinated+healthworkers+2doses, #affected+cumulative+vaccinated+healthworkers+2doses\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:32:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+healthworkers+1dose, #affected+cumulative+vaccinated+healthworkers+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses, #affected+vaccinated+healthworkers+2doses, #affected+cumulative+vaccinated+healthworkers+2doses\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+healthworkers+1dose, #affected+cumulative+vaccinated+healthworkers+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses, #affected+vaccinated+healthworkers+2doses, #affected+cumulative+vaccinated+healthworkers+2doses\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 550ff8d0-aac2-4a77-bfeb-a6da9dd8ecae, MRT_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource 928a4140-5472-458d-b23c-776a8469e62d, MLI_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource 03fcb424-4149-49c2-9a3f-84536870ad12, VE_lang_admin0.csv\n",
            "Processing 300 of 612 (49.02%) resources\n",
            "    Accessing data for resource 36bab3c0-106c-4735-baee-e21b6924d7a2, Proyectos_san_2022_hrp_hdx\n",
            "    Accessing data for resource be7434d8-1cd2-4c18-b303-ad95053170f3, victimas_explotacion_sexual_comercial.xlsx\n",
            "    Accessing data for resource a8d86cf4-53ff-4137-b7a7-361ab55b31df, violencia_sexual_desastres_naturales.xlsx\n",
            "    Accessing data for resource 55505ce8-f3b9-4b3a-b3bb-d04a577de138, UKR_ 2022 HRP_Target SADD.xlsx\n",
            "    Accessing data for resource 2537d8e4-c376-4fce-a18f-1f8b70551972, datos_brutos_personas_alcanzadas_vbg.xlsx\n",
            "    Accessing data for resource 2b50a59a-c07c-4e95-a362-a050361bca52, PAK_Consolidated_4W_Q1_Q4\n",
            "    Accessing data for resource 2857979e-a528-429e-b7ce-e4b1c3317718, Mozambique COVID-19 Cases.xlsx\n",
            "    Accessing data for resource ed2d7bba-9df6-42eb-a92f-eb7bdc49c951, 11022022_pin_target_reached.xlsx\n",
            "    Accessing data for resource 3a74c057-475b-471b-9a8c-796a0e0086f0, Government Actions on COVID-19 in Developing Countries - xls\n",
            "    Accessing data for resource 969f3105-dd01-449c-a5e1-28a3ac6fed98, Malawi  - BA Tropical Storm  Ana  - R1\n",
            "Processing 310 of 612 (50.65%) resources\n",
            "    Accessing data for resource 8bcc5c87-21e8-4102-b874-ede0c6bdda9b, PIN_SALUD_2022_HDX.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:33:51 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, 05001, 72727.56735918422, 2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, 05001, 72727.56735918422, 2\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:33:51 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, 05001, 72727.56735918422, 2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+barriers to healtcare, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, #indicador+mortality to epidemiological, 05001, 72727.56735918422, 2\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 29dd1c53-b1e7-4756-b782-09d389b422ed, Education Cluster Activities Dataset Consolidated 2019.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:34:00 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0, 42\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0, 42\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:34:00 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0, 42\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0, 42\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource e66d4346-5c48-4a2c-840a-f0ffad339318, Zim_3W_August_2021.xlsx\n",
            "    Accessing data for resource e5c97513-8597-49f4-bd0a-754165f6439a, 200402-datos-dashboard-acceso-a-servicios.xlsx\n",
            "    Accessing data for resource 1a8d10a9-4c6b-46b1-9360-81205784f268, datos-para-hpc-v2.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:34:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#value+waste1+2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #value+waste1+2\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:34:15 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#value+waste1+2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #value+waste1+2\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 9440386d-5ae0-4eba-b687-338a2ef79786, 2010-2019-consolidado-sivicap (1).xlsx\n",
            "    Accessing data for resource 90d10e73-06bf-45a2-80f6-e43068709afe, 201812-indicadores-wash-extranjeros-censo (1).xlsx\n",
            "    Accessing data for resource f06ecfb9-c5e5-427e-aa0c-365b8b485d37, base-de-datos-wash (1).xlsx\n",
            "    Accessing data for resource a06af1f0-3942-4b9c-aabc-5a5bcd0e25e7, afghan-voluntary-repatriation-2021.xlsx\n",
            "    Accessing data for resource 54cb50fa-12dd-4f1d-be5d-32f69d0a2a39, Base convalidaciones - HDX.xlsx\n",
            "Processing 320 of 612 (52.29%) resources\n",
            "    Accessing data for resource eb4fcf4a-2fe6-46cf-a1c7-2a2026a8809c, base-desercion-escolar.xlsx\n",
            "    Accessing data for resource 6ddcfbc9-fa06-4b14-b9a4-ce96d3fae65e, conectividad-internet-5-a-19-anos.xlsx\n",
            "    Accessing data for resource 05fe6261-e81d-4854-adaf-81475f9c4205, base-cobertura-educativa.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:37:13 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=coverage+percent\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: coverage+percent\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:37:13 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=coverage+percent\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: coverage+percent\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource ca4c6d23-75e2-43cb-972f-95dbe5e7ebe1, instituciones_de_salud_en_colombia.xlsx\n",
            "    Accessing data for resource 79149c89-b6b8-48cd-8808-c098956ce8d8, acciones-de-cooperacion-para-ninos-ninas-y-adolescentes-migrantes-venezolanos.xlsx\n",
            "    Accessing data for resource 0f089229-482c-4548-96f7-7eb63f714b9f, 200630_datosinfografiasocios.xlsx\n",
            "    Accessing data for resource f59fe73c-9f2c-489a-add7-4f50f27ebbd2, delito-sexual-ven-col.xlsx\n",
            "    Accessing data for resource ac90b2d0-dee4-4230-8bae-e4016a316de4, apoyos-de-la-cooperacion-con-insumos-para-salud-ante-covid-19.xlsx\n",
            "    Accessing data for resource 9229fd69-29b7-4b29-b806-1631b6a62e05, vih_sida-vf.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:14 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+density+1000, #affected+density+1000\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+density+1000, #affected+density+1000\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:14 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+density+1000, #affected+density+1000\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+density+1000, #affected+density+1000\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 8c7f094b-a78f-42dc-8ff9-0c8ca2be5771, SECOP_HDX.xlsx\n",
            "Processing 330 of 612 (53.92%) resources\n",
            "    Accessing data for resource f8c5a8e8-ccd7-4e92-9ed0-33500f6f1359, iraq_hno_2021_severity_district_20210603\n",
            "    Accessing data for resource 5db9438b-3692-4a0e-b49f-b966d96c3b41, PIN_SAN_2022_HDX.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+low birth weight, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #indicador+food security, #inneed+afro-colombian\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 1080cfda-cd4b-444a-817c-64a8bc61a1f2, Población objetivo de Salud_HDX.xlsx\n",
            "    Accessing data for resource 578b6f44-ec18-424b-8505-9711f012a7bf, Proyectos del Clúster de Salud_2022_HDX.csv\n",
            "    Accessing data for resource 7e2df627-c3b9-4b24-8030-72cfc8cbf80b, KP_Tribal_Districts_HF_Registry_v2_20180320.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:35 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sr. no., #full name of health facility, #name of health facility, #number of beds, #\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sr. no., #full name of health facility, #name of health facility, #number of beds, #\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:35 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sr. no., #full name of health facility, #name of health facility, #number of beds, #\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sr. no., #full name of health facility, #name of health facility, #number of beds, #\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 496f4397-99b0-4381-8c44-80584d1f8738, Pakistan IDPs by Area of Origin\n",
            "    Accessing data for resource 7b1e5eff-b760-4df0-9a95-2503f0cecb79, LBY_POP_2021\n",
            "    Accessing data for resource 5115e8c2-f4b4-4fa3-8d56-54c47c741f1d, PiN_WASH_2022_HDX.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:47 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+afro-colombian, #inneed+disability-condition\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+afro-colombian, #inneed+disability-condition\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:38:47 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#inneed+afro-colombian, #inneed+disability-condition\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #inneed+afro-colombian, #inneed+disability-condition\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource c408b46d-3f45-46ac-abe0-7c697a73b0db, LBY_HNO_2022.xlsx\n",
            "    Accessing data for resource 5e1bea70-be2f-4a17-95dc-1c4700cd75ba, Salud_financiamiento_FTS_HDX_2021.xlsx\n",
            "Processing 340 of 612 (55.56%) resources\n",
            "    Accessing data for resource 5722fb9c-6dd5-485f-87d6-84421b337288, UKR_Population Baseline_SADD_2022.xlsx\n",
            "    Accessing data for resource fb31450c-b6e3-4660-a75f-44fcec5e0372, Kerela.xlsx\n",
            "    Accessing data for resource cd541fbe-01f9-499d-8b88-41b84b9b596e, hdx_summary_stats_2021.csv\n",
            "    Accessing data for resource d20b1642-e6de-4bc1-9315-81a32eb88c3a, PAK_HNO_2021.xlsx\n",
            "    Accessing data for resource 92c7b3c3-c14f-4e8c-becc-b12f2f272967, Iraq Covid 19 subnational data\n",
            "    Accessing data for resource 4a3fbee8-f82b-422c-95c0-ee68be073704, DATOS_Discapacidad_HDX.xlsx\n",
            "    Accessing data for resource 745442ed-18e6-408c-8241-2db7702d7c21, SAN_Socios_2021_hdx.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:39:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#beneficiary+focus+indigenous+aro-colombian, #activity+crisis+national+covid-19\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #beneficiary+focus+indigenous+aro-colombian, #activity+crisis+national+covid-19\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:39:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#beneficiary+focus+indigenous+aro-colombian, #activity+crisis+national+covid-19\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #beneficiary+focus+indigenous+aro-colombian, #activity+crisis+national+covid-19\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 7ce27863-30e6-4de5-ba24-388855f4364b, Schoolsandtheircoordinates2020.xlsx\n",
            "    Accessing data for resource dd330c8d-f092-4750-8405-66d6b546cd62, mapeo_actores_vbg.xlsx\n",
            "    Accessing data for resource 5f144109-39c6-4b00-8a32-b983b957e65c, Partos_grupos etarios_2020.xlsx\n",
            "Processing 350 of 612 (57.19%) resources\n",
            "    Accessing data for resource 91794717-4e92-4f52-a483-3e4198f60270, PiN_VBG_HDX.xlsx\n",
            "    Accessing data for resource 40ef29d7-8097-4261-be42-186705cb7473, Tasa_Afectados_Desastres_2019_2020.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:39:51 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+total+2019, #population+total+2020, #affected+2019, #affected+2020\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+total+2019, #population+total+2020, #affected+2019, #affected+2020\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:39:51 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+total+2019, #population+total+2020, #affected+2019, #affected+2020\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+total+2019, #population+total+2020, #affected+2019, #affected+2020\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource c60fac4d-b3fb-48a7-9652-7d7e44b34a89, Health Facility List (with geo-codes).xlsx\n",
            "    Accessing data for resource 03d86390-5a57-4a14-b0d8-2288fb7dac9d, presunto-delito-sexual-sexo-edad-septiembre.xlsx\n",
            "    Accessing data for resource 4ef84622-0e2e-4251-999d-02f46c0d585c, BFA_Covid19_Citylevel_HXL_HERA.csv\n",
            "    Accessing data for resource ba97f052-78bc-471d-89d3-fe48a19b8659, MLI_Covid19_Citylevel_HXL_HERA.csv\n",
            "    Accessing data for resource 1c66a857-d11a-4519-baca-8f1f5e2ce2a4, 20210810_location_fce_rrrc_unhcr_population-registration_public.xlsx\n",
            "    Accessing data for resource b1d1f96b-624b-4275-bdab-0fe670d11299, Iraq_Population_2021_CSO_Projection.xlsx\n",
            "    Accessing data for resource 3f727fc9-9f88-4f22-934c-ce6c10db0aaa, DTM Mauritania Migrants Baseline Assessment Round 1\n",
            "    Accessing data for resource ae821d86-cb34-4421-b1c9-db7708572038, DTM Haiti - EQ(2021) - Site Assessment - Round 2\n",
            "Processing 360 of 612 (58.82%) resources\n",
            "    Accessing data for resource 9affe23f-3ce2-40d1-ae6a-1d7060b548a5, PDI_Mali_2012_2021.xlsx\n",
            "    Accessing data for resource a3950327-3445-4e6c-9a94-f6ec752cefa9, 2021-11-08 - Ukraine_Security incidents_2017-2021.xlsx\n",
            "    Accessing data for resource 01da0966-2398-4a6a-adbb-57c5f3f21583, SV_lang_admin0_v01.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:40:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_len, #indicator+lang+pct+iso639 3_ccr, #indicator+lang+pct+iso639 3_ppl, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_deu, #indicator+lang+pct+iso639 3_fra, #indicator+lang+pct+iso639 3_por, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_len, #indicator+lang+pct+iso639 3_ccr, #indicator+lang+pct+iso639 3_ppl, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_deu, #indicator+lang+pct+iso639 3_fra, #indicator+lang+pct+iso639 3_por, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:40:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_len, #indicator+lang+pct+iso639 3_ccr, #indicator+lang+pct+iso639 3_ppl, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_deu, #indicator+lang+pct+iso639 3_fra, #indicator+lang+pct+iso639 3_por, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_len, #indicator+lang+pct+iso639 3_ccr, #indicator+lang+pct+iso639 3_ppl, #indicator+lang+pct+iso639 3_eng, #indicator+lang+pct+iso639 3_deu, #indicator+lang+pct+iso639 3_fra, #indicator+lang+pct+iso639 3_por, #indicator+lang+pct+iso639 3_ita, #indicator+lang+pct+iso639 3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 6f66c234-92c9-42fb-a16e-b1a4ca0b4bb4, IRV-2021.xlsx\n",
            "    Accessing data for resource b54bf736-5f79-425b-bd94-38bb405d3593, Nigeria_3W_August_2021.xlsx\n",
            "    Accessing data for resource e1b7dbce-14c4-4f20-be37-27aa0670e3d7, Somalia drought viz - UNHCR-PRMN-displacements\n",
            "    Accessing data for resource 927ced8e-f1e0-4666-b259-af1403eb1a96, who-is-doing-what-and-where_nga_3w_august_2021.xlsx\n",
            "    Accessing data for resource a7440a59-e4ef-4050-a155-aaf332e04577, ukraine_civilian casualties_2016-2021.xlsx\n",
            "    Accessing data for resource e8c619e6-59b0-4e00-a912-76e18fe5b9f9, Nigeria Hospitals and Clinics_HXL.xlsx\n",
            "    Accessing data for resource cefed0c9-230e-4313-bbe7-25e5dc441ff9, eth_agriculture_cluster_4w_march_june_2021\n",
            "Processing 370 of 612 (60.46%) resources\n",
            "    Accessing data for resource 37565fbc-5415-45a4-9f6c-b856653dae5d, Excess mortality during COVID-19 pandemic\n",
            "    Accessing data for resource 997dc52d-0482-4773-9b8a-7e62d32e8c54, Daily-Update IDN-COVID19\n",
            "    Accessing data for resource db7589de-7ac1-4c13-9342-e27bf7449cbd, Ukraine EECPs.xlsx\n",
            "    Accessing data for resource 60505df5-26a7-4be4-9723-83d0d6c0f181, CAR_Incidents_affectant_Système_Santé_2021.xlsx\n",
            "Dataset honduras-operational-presence not found!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:30 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:30 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+vaccinated+1dose, #affected+cumulative+vaccinated+1dose, #affected+vaccinated+2doses, #affected+cumulative+vaccinated+2doses\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 63894b5f-ad22-4ac4-bda6-a35d24908ef6, DRC_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource 79eadcbd-af15-4218-862a-40714f468c1b, Haiti 3W data\n",
            "    Accessing data for resource a4ddd895-00ed-406b-ae28-c7886bade322, who-is-doing-what-and-where_nga_3w_jan_Jun_2021.xlsx\n",
            "    Accessing data for resource 58b9b864-f794-4ee4-a3f2-58827f7c9504, DTM CAR Site Assessment Round 12\n",
            "    Accessing data for resource 9b44a1a0-5d32-4969-9daa-afa5d2910052, 20210719_5w_cluster-template-cleaned-v6-quickchart\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:55 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=indicator+distributed, note, adm1_pcode, adm2_pcode, adm3_pcode_, wash activity group, ind, total reached, check org 1, check org 2, check donnors, check sites, activity check3, activity_indicator2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: indicator+distributed, note, adm1_pcode, adm2_pcode, adm3_pcode_, wash activity group, ind, total reached, check org 1, check org 2, check donnors, check sites, activity check3, activity_indicator2\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:55 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=indicator+distributed, note, adm1_pcode, adm2_pcode, adm3_pcode_, wash activity group, ind, total reached, check org 1, check org 2, check donnors, check sites, activity check3, activity_indicator2\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: indicator+distributed, note, adm1_pcode, adm2_pcode, adm3_pcode_, wash activity group, ind, total reached, check org 1, check org 2, check donnors, check sites, activity check3, activity_indicator2\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 380 of 612 (62.09%) resources\n",
            "    Accessing data for resource 0c65956d-906f-4b12-9721-eef5823a6503, EC_lang_admin0.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:59 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:41:59 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource a5905c0c-a404-4668-aa12-b4227d0d8792, 210318_OCHA 3W_COVID 19 Humanitarian Response.xlsx\n",
            "    Accessing data for resource 28732ce4-3100-4ebb-b353-3eb8fed9a56b, Sudan_Floods_Affected_Localities_10Nov2020\n",
            "    Accessing data for resource 63ef99fd-d2b5-4e56-8ecb-89f7da87a9a4, HTI_TARGET_HRP2021.xlsx\n",
            "    Accessing data for resource f4909418-05cd-45ac-9a9b-0f11830c65d8, Intersectoral severity of needs - Ethiopia 2021 HNO.xlsx\n",
            "Dataset education-in-emergency-eie-key-figures-2018-2019 not found!\n",
            "    Accessing data for resource 57501c43-83a4-4e7c-b69f-b034f65b9cd2, SUDAN_HNO 2021_Baseline Data.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:42:27 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=vulr18, u5_per, plw\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: vulr18, u5_per, plw\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:42:27 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=vulr18, u5_per, plw\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: vulr18, u5_per, plw\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource a000ceaa-70c9-4db7-8ee3-cb5c17130fae, BJ_lang_admin0.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:42:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:42:32 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource b5bda083-b26c-49a6-9aa4-f476f19f17d0, UA-IDPs (2021 HNO).xlsx\n",
            "    Accessing data for resource 7e2ec0b7-c46b-49e3-ad2e-44ba077abd23, TGO_Covid19_Citylevel_HXL_HERA.csv\n",
            "Processing 390 of 612 (63.73%) resources\n",
            "    Accessing data for resource fae45d2c-ffae-4de8-b54e-2bca673e0f42, TGO_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource e1003a2f-280b-429c-9ad8-997e30642e42, Pakistan National Nutrition Survey 2018.xlsx\n",
            "    Accessing data for resource 618c6f61-bd9c-448a-8913-5c96ba956af1, Zimbabwe_HNO.xlsx\n",
            "    Accessing data for resource 4b9ab62b-8395-4eb0-93dc-8e253b0a9898, afghanistan-3w-operational-presence-january-to-march-2021.csv\n",
            "    Accessing data for resource e21a03c5-0e7c-4a07-8292-43a26f92a48c, Yemen_CIMP - Civilian Structure\n",
            "    Accessing data for resource ad496fd4-7b76-432a-894c-6870e8844e85, Yemen_CIMP - Number of casualities\n",
            "    Accessing data for resource a10e7b0b-e14e-4536-9b82-eb93ac2f5485, Zimbabwe Baseline Population.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:01 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=15, 0, 15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 15, 0, 15\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:01 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=15, 0, 15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 15, 0, 15\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource f0acef6a-a628-4fc5-b675-eb58783b2e2a, unrwa_pse_refugees_31dec2019.xlsx\n",
            "    Accessing data for resource af6cae79-86e2-411a-9ac8-351dc3920864, 210316_3W_COVID 19 Humanitarian Response_RCCE.xlsx\n",
            "    Accessing data for resource 9a4cef39-9ac5-4fff-945c-d9098816dac9, PE_lang_admin0.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:17 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_cni, #indicator+lang+pct+iso639 3_ayr, #indicator+lang+pct+iso639 3_que, #indicator+lang+pct+iso639 3_spa, #indicator+lang+pct+iso639 3_unknown_indigenous, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_cni, #indicator+lang+pct+iso639 3_ayr, #indicator+lang+pct+iso639 3_que, #indicator+lang+pct+iso639 3_spa, #indicator+lang+pct+iso639 3_unknown_indigenous, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:17 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639 3_cni, #indicator+lang+pct+iso639 3_ayr, #indicator+lang+pct+iso639 3_que, #indicator+lang+pct+iso639 3_spa, #indicator+lang+pct+iso639 3_unknown_indigenous, #indicator+lang+pct+iso639 3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639 3_cni, #indicator+lang+pct+iso639 3_ayr, #indicator+lang+pct+iso639 3_que, #indicator+lang+pct+iso639 3_spa, #indicator+lang+pct+iso639 3_unknown_indigenous, #indicator+lang+pct+iso639 3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 400 of 612 (65.36%) resources\n",
            "    Accessing data for resource b0db3d0b-1c6c-41ec-a4c6-bbc0bc22f27c, bfa_hno_2021.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:24 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=42, 15, 0, 15, 15, 0, 15, 15, 15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 42, 15, 0, 15, 15, 0, 15, 15, 15\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:43:24 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=42, 15, 0, 15, 15, 0, 15, 15, 15\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 42, 15, 0, 15, 15, 0, 15, 15, 15\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource de3d30de-016f-4b10-9711-9fbd1846bf92, Myanmar_HNO_HRP_2021-hxl.xlsx\n",
            "    Accessing data for resource 4b95538b-0fc8-4870-8cfd-8f08a2d2f1d3, LIF_3w_DatafromNov2020HXL.xlsx\n",
            "    Accessing data for resource 95a4d096-4480-48f7-85d4-310bf91c43c8, Burundi Cankuzo_FD.xlsx\n",
            "    Accessing data for resource 28cc6655-2f83-43ba-8338-85d4bc6ac97f, 2016-2020 Air- and ground-launched explosive weapons affecting health facilities.xlsx\n",
            "    Accessing data for resource efef2f04-b446-4963-bf27-cf9af9fc8d3a, SEN_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource c279e1ea-9e29-45e2-94b7-b51d389e8ab6, tcd_hrp_2021.xlsx\n",
            "    Accessing data for resource 9fc0df81-bbb1-43f5-beb9-45b625fa3aa7, Site Assessment Round 2 - Pb\n",
            "    Accessing data for resource bdfd19ae-e815-4935-95ce-9dcbddf4e40b, 06_IOM DTM Dataset Kenya_Round 1_20150530_public_0_0.xlsx\n",
            "    Accessing data for resource 12ad71a7-0f51-4548-a621-f0a9d9a4e5e7, Ecuador_site_assessment_R7_Pb\n",
            "Processing 410 of 612 (66.99%) resources\n",
            "    Accessing data for resource b5a9fe30-0e20-4919-9552-7e1e6ed66608, etm-cycle-1-non-confidential-dataset-20160327\n",
            "    Accessing data for resource f83b2f21-027d-4752-af7f-45f11f7293ef, etm-cycle-2-complete-dataset-sanitized\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:44:10 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#geo=lon\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #geo=lon\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:44:10 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#geo=lon\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #geo=lon\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource e9f78e16-ee45-4070-bdee-a2050e66ba20, afghanistan-3w-operational-presence-january-to-march-2020.csv\n",
            "    Accessing data for resource d554c9dc-62c2-4ad4-8e1c-a04b4fd03f57, 2020 SHCC Health Care Nagorno-Karabakh Data.xlsx\n",
            "    Accessing data for resource 77979098-1005-457f-b23b-708c7f12be26, DRC HNO 2021_People in Need.xlsx\n",
            "    Accessing data for resource cf6f622e-a904-47f8-897e-a500507c70b4, south_sudan_2021_humanitarian_needs_overview.xlsx\n",
            "    Accessing data for resource 7106f14e-61f5-41d6-bc0f-6432edafeebb, CAF_HRP_2021\n",
            "Error reading dataset mozambique-hno ... Failed when trying to read: id=mozambique-hno! (POST)\n",
            "    Accessing data for resource ce5c0b0b-4e91-438a-bc49-f08ac7f88b13, 210330_Typhoon Goni (Rolly) and Vamco (Ulysses)_3W response.xlsx\n",
            "    Accessing data for resource 3ea2cad4-13db-4099-8998-76a5a56f0640, 2020-HRP-Sectors-Response-Jan-Dec-HXLV2.xlsx\n",
            "Processing 420 of 612 (68.63%) resources\n",
            "    Accessing data for resource 8f7ec7f6-7b26-4a20-9a42-edae588bdafe, Guinea_Ebola_National_2021_HXL_HERA.csv\n",
            "    Accessing data for resource 5fe242d8-e62a-4c23-96a9-4a8d9f8b001b, Guinea_Ebola_Subnational_2021_HXL_HERA.csv\n",
            "    Accessing data for resource 8f3f7548-3feb-43e1-b9f3-780332eb0e67, Access Indicators\n",
            "    Accessing data for resource f8dbbb79-8e47-4e60-b54a-5e502211a932, covid_data_moscow_region_weekly_2021\n",
            "    Accessing data for resource 653f5bad-0bea-44f2-991f-010a6d5efbba, Central_African_Republic_Population_Projection_2021\n",
            "    Accessing data for resource 32ba39ee-62a9-4b78-882d-ce7701a6113b, ml_lang_v01.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:45:02 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_bam, #indicator+lang+pct+iso639-3_ful, #indicator+lang+pct+iso639-3_hmb, #indicator+lang+pct+iso639-3_snk, #indicator+lang+pct+iso639-3_kao, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mey, #indicator+lang+pct+iso639-3_tmh, #indicator+lang+pct+iso639-3_rkm, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_bxw, #indicator+lang+pct+iso639-3_bze, #indicator+lang+pct+iso639-3_ara, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_bam, #indicator+lang+pct+iso639-3_ful, #indicator+lang+pct+iso639-3_hmb, #indicator+lang+pct+iso639-3_snk, #indicator+lang+pct+iso639-3_kao, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mey, #indicator+lang+pct+iso639-3_tmh, #indicator+lang+pct+iso639-3_rkm, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_bxw, #indicator+lang+pct+iso639-3_bze, #indicator+lang+pct+iso639-3_ara, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:45:02 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_bam, #indicator+lang+pct+iso639-3_ful, #indicator+lang+pct+iso639-3_hmb, #indicator+lang+pct+iso639-3_snk, #indicator+lang+pct+iso639-3_kao, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mey, #indicator+lang+pct+iso639-3_tmh, #indicator+lang+pct+iso639-3_rkm, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_bxw, #indicator+lang+pct+iso639-3_bze, #indicator+lang+pct+iso639-3_ara, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_bam, #indicator+lang+pct+iso639-3_ful, #indicator+lang+pct+iso639-3_hmb, #indicator+lang+pct+iso639-3_snk, #indicator+lang+pct+iso639-3_kao, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mey, #indicator+lang+pct+iso639-3_tmh, #indicator+lang+pct+iso639-3_rkm, #indicator+lang+pct+iso639-3_myk, #indicator+lang+pct+iso639-3_bxw, #indicator+lang+pct+iso639-3_bze, #indicator+lang+pct+iso639-3_ara, #indicator+lang+pct+iso639-3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 6e750e3b-28f8-4846-b512-a35450faa36f, 4W_BU_Kayanza.csv\n",
            "    Accessing data for resource 1e6050b6-2bf6-4050-aed8-7157f2c3cc5c, DR Congo Covid 19 Subnational cases\n",
            "    Accessing data for resource 2a816dc6-d079-4db8-b8fc-84458b006491, 28Jan21 - 5W & Assesment - OCHAIDN.xlsx\n",
            "    Accessing data for resource 74f86547-b489-45d5-a613-15d56dff734d, 4W_BU_Kirundo.csv\n",
            "Processing 430 of 612 (70.26%) resources\n",
            "    Accessing data for resource 17f5ddef-ba47-47fa-8108-c564c5da5385, fieldsdata_4w_UG_BujumburaMarie.csv\n",
            "    Accessing data for resource 6510f20d-f959-49dc-a544-b9dfa4c7561b, Ethiopia_Covid19_cases_HXL_HERA.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:45:22 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=ethiopia\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: ethiopia\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:45:22 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=ethiopia\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: ethiopia\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 41a5a96e-bf08-4228-bcc8-9229cd7c7b4d, DTM Armenia Baseline Assessment Round 3\n",
            "    Accessing data for resource bdab79f3-9f24-4dbb-99a2-637375ac97bd, Italy - Food Prices\n",
            "    Accessing data for resource f77153af-56f7-49d2-abf0-8c4598eb46b4, cleaned_compiled_Nov_3ws.xlsx\n",
            "    Accessing data for resource be59bc8b-11a2-4430-bbc2-dd7584d9cf06, DTM Guatemala Site Assessment ETA_IOTA R1\n",
            "    Accessing data for resource 4f44557e-e5e8-4892-9984-247d0fbd1ea8, DTM Honduras Site Assessment  Hurricane ETA And IOTA Response R 1\n",
            "    Accessing data for resource db57c14c-514d-4744-8079-0bbaa2879c45, ni_lang_v01_admin0.csv\n",
            "    Accessing data for resource 866f4dcc-b1de-4714-ae12-b2c4a3b25b56, npm-site-assessment-round-16-dataset-20191010\n",
            "    Accessing data for resource 5ee2111a-86bc-4ce2-b127-14f0bd4d5482, LCB_SnapShot_DataSets - key_figures.csv\n",
            "Processing 440 of 612 (71.90%) resources\n",
            "    Accessing data for resource 2ff8c88a-c7d9-4e2c-a624-fb9ea8c4c941, Bénin_Covid-19_Subnational_HXL.csv\n",
            "    Accessing data for resource a34a830a-7446-43d1-b7d2-9ddb38c10f7e, SDN_HAC_Flood data by state and locality_07102020_hxl\n",
            "    Accessing data for resource 5ae69dc0-85d2-4684-a8c8-ec0b2ac3c106, GMB_Subnational_Covid19_HXL_HERA.csv\n",
            "    Accessing data for resource 3d811952-8651-4222-891c-f7ab607bbb47, fts_requirements_funding_esh.csv\n",
            "    Accessing data for resource 8a75dfd8-93ee-42ea-ad28-f5f77b898120, DTM Nepal Baseline Assessment Floods20 R1\n",
            "    Accessing data for resource 2817191d-40b6-4588-917b-2aeced2cb7e2, DTM Sri Lanka Baseline Assessment Round 2\n",
            "    Accessing data for resource dfea9ccb-fb06-419e-8685-19f480dd2800, DTM Bolivia Site Assessment Round 1\n",
            "    Accessing data for resource 6ef172d0-9b44-4d16-ba3c-17a709b6ea44, DTM Antigua And Barbuda Site Assessment Round 1\n",
            "    Accessing data for resource 93522425-22ff-410b-8cb4-8bdb566274a5, Humanitarian Access August 2020\n",
            "    Accessing data for resource c245de0c-500d-45ea-9d71-40372352174f, pblStatsSum\n",
            "Processing 450 of 612 (73.53%) resources\n",
            "    Accessing data for resource dc9f6492-5a7e-4ffb-9114-ef994ab8ae6c, MALI_HPC_2020.xlsx\n",
            "    Accessing data for resource 8c4c3b57-cd53-4b2b-a882-86760e258038, PNG_covid_3w_data_aug2020.xlsx\n",
            "    Accessing data for resource 42c867d4-3e81-430e-8db2-1c8f36f00987, Mali covid-19 data\n",
            "    Accessing data for resource 0dc923ce-190e-483f-ac80-3de6d7f06b85, DRC HRP 2020\n",
            "    Accessing data for resource 4532a799-ebaf-4ff6-95a5-3607973f0de4, ebola-epidemic-national-figures.csv\n",
            "    Accessing data for resource 5a2b4043-08c8-47e5-8294-8118c6985520, time_series_covid19_confirmed_global_iso3_regions.csv\n",
            "    Accessing data for resource 13243fb8-1679-496c-aa59-0edaf25dafe5, DTM El Salvador Site Assessment Round 2\n",
            "    Accessing data for resource e21ab62f-2b89-4034-a57d-0d226a2e36b9, Guinea_Covid-19_(Sub)national_HXL.csv\n",
            "    Accessing data for resource 277d4970-e5c3-444b-ba07-61d8d5fa0101, Liberia_Covid-19_Subnational_HXL.csv\n",
            "    Accessing data for resource 6c1fff73-a418-4f2c-86ca-748019180501, 2018_Prevalence_Of_Diseases_Outpatient_clinics_per_state.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+diseases_of_ respiratory_system\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+diseases_of_ respiratory_system\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#affected+diseases_of_ respiratory_system\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #affected+diseases_of_ respiratory_system\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 460 of 612 (75.16%) resources\n",
            "    Accessing data for resource 57a4d991-f760-472f-9312-868c10498bc1, co_lang_v01.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_emp, #indicator+lang+pct+iso639-3_pbb, #indicator+lang+pct+iso639-3_bpb, #indicator+lang+pct+iso639-3_guc, #indicator+lang+pct+iso639-3_cto, #indicator+lang+pct+iso639-3_spa, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_fra, #indicator+lang+pct+iso639-3_ita, #indicator+lang+pct+iso639-3_deu, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_emp, #indicator+lang+pct+iso639-3_pbb, #indicator+lang+pct+iso639-3_bpb, #indicator+lang+pct+iso639-3_guc, #indicator+lang+pct+iso639-3_cto, #indicator+lang+pct+iso639-3_spa, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_fra, #indicator+lang+pct+iso639-3_ita, #indicator+lang+pct+iso639-3_deu, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:25 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_emp, #indicator+lang+pct+iso639-3_pbb, #indicator+lang+pct+iso639-3_bpb, #indicator+lang+pct+iso639-3_guc, #indicator+lang+pct+iso639-3_cto, #indicator+lang+pct+iso639-3_spa, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_fra, #indicator+lang+pct+iso639-3_ita, #indicator+lang+pct+iso639-3_deu, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_emp, #indicator+lang+pct+iso639-3_pbb, #indicator+lang+pct+iso639-3_bpb, #indicator+lang+pct+iso639-3_guc, #indicator+lang+pct+iso639-3_cto, #indicator+lang+pct+iso639-3_spa, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_fra, #indicator+lang+pct+iso639-3_ita, #indicator+lang+pct+iso639-3_deu, #indicator+lang+pct+iso639-3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource b08078d9-2e5c-4d32-a430-733e1e9e6ed2, mhu-jan-dec2019-idn-hxl.csv\n",
            "    Accessing data for resource 6c7ee710-ea84-4cc9-981b-81c8f432bd04, kh_lang_v01.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:34 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_krr, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_pnx, #indicator+lang+pct+iso639-3_brb, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_krr, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_pnx, #indicator+lang+pct+iso639-3_brb, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:34 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_krr, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_pnx, #indicator+lang+pct+iso639-3_brb, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_krr, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_pnx, #indicator+lang+pct+iso639-3_brb, #indicator+lang+pct+iso639-3_umh, #indicator+lang+pct+iso639-3_khm, #indicator+lang+pct+iso639-3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource dd079132-bc41-4ac2-b4e7-92894277d7de, HPC 2020 sector PINs Targets final.xlsx\n",
            "    Accessing data for resource e0da0453-eae1-412e-a84b-016c8e87b758, PiN, IDPs, Refugees & Returnees 2020 figures\n",
            "    Accessing data for resource a6eea2b2-0ff8-41ae-b25e-087566f0f221, Imperial_COVID-19_Projections.xlsx\n",
            "    Accessing data for resource 9e945b6f-c825-41a6-b1ab-eed971fd792d, sudan-people-reached-by-state-jan-dec-2018_hrp.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=4000000\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 4000000\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:47:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=4000000\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 4000000\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource cdfcaf02-56ed-4dfe-914a-7a40ee5090ca, sudan-people-reached-by-locality-jan-dec-2019_hrp.xlsx\n",
            "    Accessing data for resource 7153d0f0-56b4-4282-aabc-3a0695812c0e, us-states-hxl.csv\n",
            "    Accessing data for resource 3661aacd-d9b7-4eb0-8167-681ac13d4042, th_lang_v01.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:02 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_tha, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mfa, #indicator+lang+pct+iso639-3_mya, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_tha, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mfa, #indicator+lang+pct+iso639-3_mya, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:03 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_tha, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mfa, #indicator+lang+pct+iso639-3_mya, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_unknown\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_tha, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+pct+iso639-3_mfa, #indicator+lang+pct+iso639-3_mya, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_unknown\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 470 of 612 (76.80%) resources\n",
            "    Accessing data for resource b5d618bd-f363-4240-8ce2-1536beda9299, Global Coordination Groups (Beta) CSV\n",
            "    Accessing data for resource 3cac22e4-1e09-45b5-93d5-720771161b1e, DHS Quickstats Data for Turkey\n",
            "    Accessing data for resource de4a6b23-45f6-46cc-bb68-36c59b043219, DHS Quickstats Data for Turkey\n",
            "    Accessing data for resource 742b08fe-c53d-4538-81fc-b30a41a1280a, Distribution of Population by Sex, Number of Households, Land Area, Population Density and Sub County.xlsx\n",
            "    Accessing data for resource 7267b407-3806-448c-8f78-f9b98cb8ed11, 2020_DPRK_N&P_Overview_Provisional_data.xlsx\n",
            "    Accessing data for resource b9465fc1-15d4-462e-91df-2f2c39ee5cc3, Sudan Disease Outbreaks 2019.xlsx\n",
            "    Accessing data for resource a2a23a36-cc7a-4c0c-aeb6-cb879578f8fe, spcf-31dec2019_rosea_hxl.xlsx\n",
            "    Accessing data for resource e2781ffd-9989-4112-9154-6b8d29464b11, 200304_3W on NCDDS_EQ.xlsx\n",
            "    Accessing data for resource 1e7e721e-e7bd-4508-888a-13f57301a2b6, 200303_marawi_conflict_3w.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=1\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 1\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:37 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=1\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 1\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 13ae3804-14ea-44a8-9711-f738e82afcd6, 200210_Taal Eruption 3W_Consolidated.xlsx\n",
            "Processing 480 of 612 (78.43%) resources\n",
            "    Accessing data for resource 8d45c271-3394-4860-961b-0231b91f8a09, MW_lang_V02.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_nya, #indicator+lang+pct+iso639-3_ngl, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_lai, #indicator+lang+pct+iso639-3_ngo, #indicator+lang+pct+iso639-3_nyy, #indicator+lang+pct+iso639-3_por, #indicator+lang+pct+iso639-3_seh, #indicator+lang+pct+iso639-3_nse, #indicator+lang+pct+iso639-3_tum, #indicator+lang+pct+iso639-3_toh, #indicator+lang+pct+iso639-3_yao, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+understand+pct+iso639-3_abc\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_nya, #indicator+lang+pct+iso639-3_ngl, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_lai, #indicator+lang+pct+iso639-3_ngo, #indicator+lang+pct+iso639-3_nyy, #indicator+lang+pct+iso639-3_por, #indicator+lang+pct+iso639-3_seh, #indicator+lang+pct+iso639-3_nse, #indicator+lang+pct+iso639-3_tum, #indicator+lang+pct+iso639-3_toh, #indicator+lang+pct+iso639-3_yao, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+understand+pct+iso639-3_abc\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:48:49 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+lang+pct+iso639-3_nya, #indicator+lang+pct+iso639-3_ngl, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_lai, #indicator+lang+pct+iso639-3_ngo, #indicator+lang+pct+iso639-3_nyy, #indicator+lang+pct+iso639-3_por, #indicator+lang+pct+iso639-3_seh, #indicator+lang+pct+iso639-3_nse, #indicator+lang+pct+iso639-3_tum, #indicator+lang+pct+iso639-3_toh, #indicator+lang+pct+iso639-3_yao, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+understand+pct+iso639-3_abc\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+lang+pct+iso639-3_nya, #indicator+lang+pct+iso639-3_ngl, #indicator+lang+pct+iso639-3_eng, #indicator+lang+pct+iso639-3_lai, #indicator+lang+pct+iso639-3_ngo, #indicator+lang+pct+iso639-3_nyy, #indicator+lang+pct+iso639-3_por, #indicator+lang+pct+iso639-3_seh, #indicator+lang+pct+iso639-3_nse, #indicator+lang+pct+iso639-3_tum, #indicator+lang+pct+iso639-3_toh, #indicator+lang+pct+iso639-3_yao, #indicator+lang+pct+iso639-3_unknown, #indicator+lang+understand+pct+iso639-3_abc\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 3ce474ea-8a18-4909-929f-131ac883acf8, WCA UN agencies location.xlsx\n",
            "    Accessing data for resource b4fead08-f224-407a-9c7f-bd2f9983273d, afghanistan-3w-operational-presence-october-to-december-2019.csv\n",
            "    Accessing data for resource f1aabdf9-2bbe-4b7f-b19e-eb38c07f915d, DTM Yemen Flow Monitoring - Jan19 to Dec19\n",
            "    Accessing data for resource 15e24633-67f1-463b-ad41-9b89b9dbbd07, Philippines Mindanao Earthquakes Site Assessment R5\n",
            "    Accessing data for resource d4e67ae7-a910-47c9-b283-304877c7e5eb, Humanitarian Response Plans\n",
            "    Accessing data for resource 9963ba32-bd72-4c25-96cd-d28ad9cd7e75, DTM CAR Bangui Floods Oct19 Site Assessment\n",
            "    Accessing data for resource e8a121b9-eb74-498f-91a4-8c6601eddc98, water quality data_v2.xlsx\n",
            "    Accessing data for resource 038f595a-baaf-4fa2-9b4e-04f7f901ba42, DTM_Bahamas_MSLA_Data_Round_3\n",
            "    Accessing data for resource eae0a3d3-a630-4412-ab2a-7b61046f57cc, Somalia OCHA - Monitoring Matrix 2017\n",
            "Processing 490 of 612 (80.07%) resources\n",
            "    Accessing data for resource 47a04948-0f01-4ebc-b873-dc20c7906bd7, DTM South Sudan Village Assessment May-16 to Nov-19\n",
            "    Accessing data for resource 4dd1b93d-fb52-4af3-9e92-5feaa4c5ab00, DTM Mozambique Multi Sectoral Assessment R 7\n",
            "    Accessing data for resource 37a34cf3-1994-4299-874d-e2625c37dd28, DTM Chad Assessment Return Logone Oriental Nov19 R2\n",
            "    Accessing data for resource 60df02d8-9a43-4c66-b80e-80b7aabeb2d9, DTM Chad Assessment Return Moyen Chari Nov19 R2\n",
            "    Accessing data for resource adff7da9-3486-40af-849a-c5119b8e0bfd, DTM Mozambique Baseline Assessment Round 6\n",
            "    Accessing data for resource 45684efe-2a2c-4218-97a8-e5c11bd3d3d0, Functional Roles (Beta) - CSV\n",
            "    Accessing data for resource 7032dafc-713d-4969-b2f7-44b561836346, DI_Stat1129_GeoDist_HXL.xls\n",
            "    Accessing data for resource 2baae404-fd46-4c74-9d92-0fa8231d0cb7, DI_Stat1132_GeoDist.xls\n",
            "    Accessing data for resource 85d9a279-7231-4e09-915c-c46133749167, ethiopia-3w-operational-presence-september-2019_hxl.xlsx\n",
            "    Accessing data for resource f52cb883-c118-43d8-b59e-811e0c7a46a7, afghanistan-natural-disaster-incidents-from-1-january-to-31-dec-2016.xlsx\n",
            "Processing 500 of 612 (81.70%) resources\n",
            "    Accessing data for resource 274fd343-8adb-4ef8-b236-bed6037d097c, 190409_3W_Mangkhut_Yutu_April2019_Final.xlsx\n",
            "    Accessing data for resource d014c8c8-e6b5-48fb-a6d3-79f1a44a04a3, 190910_3W for Mindanao Conflict (excluding Marawi).xlsx\n",
            "    Accessing data for resource c4c65bc5-afac-451b-9082-9a0107ea08cf, Marawi Conflict IDPs_29June.xlsx\n",
            "    Accessing data for resource 7770c0a6-e6d9-4607-b6f2-70259b886180, phl-marawi_idp-by-month_region.csv\n",
            "    Accessing data for resource 3f660e0d-5667-4a5c-a5c8-bfb73bf9b8e9, afghanistan-natural-disaster-incidents-from-1-january-to-31-december-2015.xlsx\n",
            "    Accessing data for resource 903326f2-b372-4786-9973-87226cb15e41, Pin trend 3 (1).csv\n",
            "    Accessing data for resource 90ee46eb-ab51-4095-8c32-b9f8361364b5, DTM Sudan Registration\n",
            "    Accessing data for resource 81afa4e3-204e-4ff6-bc86-87615419ffd2, DTM DRC BA Lomami Oct2018 R2\n",
            "    Accessing data for resource b826a603-f72f-4b52-9600-9f8725abd8ce, CP_5W_Data_cleaned_HDX.xlsx\n",
            "    Accessing data for resource d1fa03ee-ad6d-4065-bcd8-fe45b9d9cf66, 4W-Moz-16062019.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:50:57 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=comments, org type verified?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: comments, org type verified?\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:50:57 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=comments, org type verified?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: comments, org type verified?\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 510 of 612 (83.33%) resources\n",
            "    Accessing data for resource cb10bb18-bef3-4040-ac01-4008a2677100, DTM Burkina Faso Site Assessment Round 1\n",
            "    Accessing data for resource fcc28514-36c3-4aab-91e9-1719ff0a33c2, Papua New SA Ulawun Valcona R2\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:51:03 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:51:03 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=0, 0\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: 0, 0\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource d21f0711-2722-42ee-aa1c-6dda74287ff4, DTM Malawi Site Assessment Round 3\n",
            "    Accessing data for resource 50fa214d-06bb-469c-8482-e448944c00dc, sanitation_access_opendefecation.xlsx\n",
            "    Accessing data for resource 58542259-91b3-4174-a8fe-3192e3f6511e, yemen_hard_to_reach_districts_april_2019.xlsx\n",
            "    Accessing data for resource e142446b-cfc1-40bf-9714-6dc8c0a97277, Base Desaparecidos(INML)EneroaAbril2019Elaborado08072019.xlsx\n",
            "    Accessing data for resource d6339726-55f5-47b5-9e27-11f015ade281, CXB-Education-Sector-Facility-2018-06-24.xlsx\n",
            "    Accessing data for resource c40c0d0f-562b-4912-98f4-8e629545e266, afghanistan-3w-operational-presence-january-to-march-2019.xlsx\n",
            "    Accessing data for resource a1fba5e5-1cbc-4e24-b25f-6ece34766ef0, DTM Afghanistan IDPs Returnees Needs Assessment May-Jun-18\n",
            "    Accessing data for resource cd02b8df-2cb9-4ca2-abba-ff73f31615a6, afg_casualties_2018.xlsx\n",
            "Processing 520 of 612 (84.97%) resources\n",
            "    Accessing data for resource 0faa2a30-9185-4ffd-98ed-8e42e7284caf, edu5w0117.xlsx\n",
            "    Accessing data for resource ced327ab-2441-4b17-9820-384d53449d9d, beneficiaries_reached_edu_prog_besieged 2017-2018.csv\n",
            "    Accessing data for resource fee52c29-672a-4f1b-9910-fc08452f75e5, Ethiopia East_West- Wellega RRA SA R 2\n",
            "    Accessing data for resource e51eef69-e075-442c-87e6-96d139d615a1, bmu_gazetteer.xlsx\n",
            "    Accessing data for resource cbbb2b0e-c6b2-4d12-826c-213cc0d0fdb4, idmc_displacement_all_dataset2.xlsx\n",
            "    Accessing data for resource 8b0feea6-20e4-45dc-aaae-c8b6fbd5a9f4, bfa_pin_2019.xlsx\n",
            "    Accessing data for resource 1785ccdb-794a-476f-a7c2-bda0eb934c4a, 5w_cmr_cwg_data_collect_July-to-Dec_2018.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:52:06 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=household_size, partenaires d'implementation, service financier, conditionality, date_ordre_end, mois\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: household_size, partenaires d'implementation, service financier, conditionality, date_ordre_end, mois\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:52:06 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=household_size, partenaires d'implementation, service financier, conditionality, date_ordre_end, mois\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: household_size, partenaires d'implementation, service financier, conditionality, date_ordre_end, mois\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 89dacb2d-9cb9-492a-9db3-2d997ca4267c, All sectors consolidated-ver2.xlsx\n",
            "    Accessing data for resource 68328f42-9276-423e-80d0-fe89630804ff, 3w_hxl.xlsx\n",
            "    Accessing data for resource dc003506-2ddf-4fdf-8c6d-0bb783af5390, COD_MLI.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:52:20 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=shape_length, shape_area\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: shape_length, shape_area\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:52:20 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=shape_length, shape_area\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: shape_length, shape_area\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 530 of 612 (86.60%) resources\n",
            "    Accessing data for resource cf031d00-b817-4f6f-ac58-6bf74a000cb6, Hurricane_Michael_Twitter_Data_Analysis_SummaryTable.xlsx\n",
            "    Accessing data for resource 60247249-5427-4634-93f2-0068c1a5b1a1, Dashboard-reowa-sahel-2018.xlsx\n",
            "    Accessing data for resource 05d6134c-1234-4857-982c-a9ccf42711d8, moz_cycloneidai_aerialsurvey_hxl.xlsx\n",
            "    Accessing data for resource 66ecc982-bc91-45bb-b400-49295fd7f1e8, DTM Vanuatu Manaro Round 4 Displacement-Return Dataset (2019-02).xlsx\n",
            "    Accessing data for resource 40b9b069-9695-457f-81d4-0b8d975f4cbb, DTM Yemen Area Assessment Round 37\n",
            "    Accessing data for resource ecc21f95-cfed-47a9-ab5b-10e0f81fcd19, mhr_diffa_fevrier_2019.csv\n",
            "    Accessing data for resource f6e65456-b147-45d3-bb89-848919aa0ada, MLI_DATA_HumanitarianAccessByCommuneSurveyData_20190314.xlsx\n",
            "    Accessing data for resource ee6b4c0c-6d2a-4d87-9f73-4ad34e79c66c, Haiti Ciblage HRP 2019-2020.xlsx\n",
            "    Accessing data for resource 226f8c83-1897-4273-b9bf-0dd614b6d130, hno-2019-baseline-data-hxl.xlsx\n",
            "    Accessing data for resource 99384e54-518a-4d6e-976b-f7663beb008f, hrp-nigeria-2019.xlsx\n",
            "Processing 540 of 612 (88.24%) resources\n",
            "    Accessing data for resource 37f8465f-9f6b-46cf-a8b6-d6a8a7e47166, CAR SADD 2019.xlsx\n",
            "    Accessing data for resource fce6e3eb-5ed3-4645-84e3-b45fdf20dd78, SiND SexualViolence Dataset 2015.xlsx\n",
            "    Accessing data for resource 28c478f0-50c8-4b48-a78a-3db74a135888, DTM Indonesia Sulawesi Earthquake Round 2 Site Assessment.xlsx\n",
            "    Accessing data for resource 6af28f7c-6426-46ab-a3dd-72eb1bc43729, DTM Indonesia Lombok Earthquake Round 2 Site Assessment\n",
            "    Accessing data for resource bd289ad6-9c7e-4597-bafd-c64a093f989b, Afghanistan SRSG report 2015-2017.csv\n",
            "    Accessing data for resource 890f33e1-5637-4308-95b3-355733281316, DTM Guatemala Site Assessment Round 2\n",
            "    Accessing data for resource 127b9c00-509f-4d30-8a69-af57725567ee, Lao People's Democratic Republic - Food Prices\n",
            "    Accessing data for resource c7aaaef6-a0f8-4394-b60f-06098235fbae, world-humanitarian-data-and-trends-2018_full-dataset.xlsx\n",
            "    Accessing data for resource d734df32-bd40-47d2-8bf6-8ee3ea8b457c, Maria_Text_Image_Data_Analysis_SummaryTable.xlsx\n",
            "    Accessing data for resource 0630a044-e99a-4279-a707-fb4dca67e253, All hospitals by ownership.xlsx\n",
            "Processing 550 of 612 (89.87%) resources\n",
            "    Accessing data for resource 1744dd52-f3db-47be-bed9-d09507f0107a, cmr_lt_nw_ou_sw_displaced_pop_2018-09_v01.xlsx\n",
            "    Accessing data for resource 2785704c-a961-477a-b584-00277de6dc9b, sadc-rvaa-estimated-population-2018.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:54:09 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+rural+2018\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+rural+2018\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:54:09 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+rural+2018\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+rural+2018\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 3b4b4e12-860f-4223-8d98-b756c081ec96, homicides_venezuela_2017_1.xlsx\n",
            "    Accessing data for resource d9e2e643-87b6-4aeb-a636-557b751df553, ebola-cases-and-deaths-national-may-2018-outbreak-equateur.csv\n",
            "    Accessing data for resource f54a1612-aec3-41ad-88f0-231afaf7e150, Indicators_data_ZW\n",
            "Error accessing data for resource f54a1612-aec3-41ad-88f0-231afaf7e150, Indicators_data_ZW ... list index out of range\n",
            "    Accessing data for resource f9579315-7dd0-4155-aa74-d514258046ae, Crime Trends and Operations of Criminal Justice Systems (UN-CTS).csv\n",
            "    Accessing data for resource 56a9595e-5a09-4b60-8cb2-f73492fe69af, Yemen Cholera Outbreak Epidemiology Data\n",
            "    Accessing data for resource f8d48ac8-cef8-4404-8e37-bfe8f85bce89, idp_flowdata_july_2018_finalshare.xlsx\n",
            "    Accessing data for resource 1c0cbace-7206-42fa-a84b-c5cf84e1a1de, 160704_5W_HDX.xlsx\n",
            "    Accessing data for resource bb97c4ef-7222-45b6-934b-df0a07ba011b, unicef-esaro-regional-refugee-and-idp-db-2018-august-13.xlsx\n",
            "Processing 560 of 612 (91.50%) resources\n",
            "    Accessing data for resource 999ddda6-6a24-44a9-9a49-facdd32a215b, 160516_5W_ForHDX.xlsx\n",
            "    Accessing data for resource d7fffd19-4bd0-46a5-88e0-e8e17899ea08, Redhum-Ec 5w 2.0 Ronda 12 Versión 2061012-HDX.xlsx\n",
            "    Accessing data for resource 599be5e3-35ad-4352-8fa8-377b085ab861, urban_consumption.csv\n",
            "    Accessing data for resource cdbee97a-e605-4572-bfd2-c17888aa9b07, nepal-reconstruction-survey-2018 - Recontruction_FSL_Protection_May_2018.csv\n",
            "    Accessing data for resource b383fecf-a068-4b32-8694-eb8919296b24, r58-hrrp-national-4w-180625-public.xlsx\n",
            "    Accessing data for resource cd9b4dbd-bc89-46f8-a630-90a504f79673, afghanistan-3w-april-to-june-2018_2.xlsx\n",
            "    Accessing data for resource 5588cc51-f81a-44f8-9d55-50a5007e1d8c, SOS KD WCAF outreach in 2017.xlsx\n",
            "    Accessing data for resource 92128af6-6a4d-447c-bb9f-fa67fb0e17bb, Ebola - West Africa  - Ebola Treatment Centres, Isolation Wards Hospitals and Transit Centres\n",
            "    Accessing data for resource 86babc61-4fa6-40b7-b899-36b1f167d5bc, Ecuador Earthquake damage\n",
            "    Accessing data for resource 322af5fe-8860-4c09-81c7-78510da9a4b0, Zika Cases\n",
            "Processing 570 of 612 (93.14%) resources\n",
            "    Accessing data for resource e1420f63-7b74-4e44-aa43-f03170689c4f, rainfall_adm3_oct3to5.xlsx\n",
            "    Accessing data for resource 2fba4725-41e1-4d50-8d8b-b511fd155743, Communes d'intervention CRS Senegal.xls\n",
            "    Accessing data for resource af8d8c24-e947-4472-8c1b-d12cb5647495, affected-schools-update-september-2017.xlsx\n",
            "    Accessing data for resource aea0e67a-b129-4f0a-8d20-e46aa5532fb1, Hurricane_harvey_twitter_text_data_processed_by_AIDR.xlsx\n",
            "    Accessing data for resource 9280f4ff-c43e-4f5f-8689-4e93d7f1c6e0, CH_Mali_mars_2017_HDX.xlsx\n",
            "    Accessing data for resource 8e837f63-d4c2-44bb-860b-37b3449e19a9, 3W Honduras\n",
            "    Accessing data for resource cec29b2d-32a4-439d-8b5d-5e3f14df52ed, CRS_ML_2018_Reponse_CRS_Suite_aux_Evaluations_Rapides_KISILI_2.xlsx\n",
            "    Accessing data for resource b8d1e28d-7840-45fa-8320-4a80b7ac261a, nepal-reconstruction-survey-september-2017.xlsx\n",
            "    Accessing data for resource 4f727349-9638-4fdd-9ce3-10d96b299c58, wca_Cases2017_RegCholeraPlatformWCA.xlsx\n",
            "Error accessing data for resource 4f727349-9638-4fdd-9ce3-10d96b299c58, wca_Cases2017_RegCholeraPlatformWCA.xlsx ... list index out of range\n",
            "    Accessing data for resource fb562af6-9c94-4625-9a0d-c45f70145169, DTM Burundi Site Assessment Round 26\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#adm2+ code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #adm2+ code\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:18 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#adm2+ code\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #adm2+ code\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing 580 of 612 (94.77%) resources\n",
            "    Accessing data for resource 25b04cb5-53c0-43f7-8b7e-898ff7b34c2b, Somalia flood - people affected per district\n",
            "    Accessing data for resource fe1cbb23-fb92-4daf-8555-2862d289ae53, wca_Cases2015_RegCholeraPlatformWCA.xlsx\n",
            "    Accessing data for resource 13708985-49a3-4d3b-aec5-315628ee2ec4, cmr_car_refugees_20171231_unhcr_hxl_v1.1.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:27 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+refugees+female+age>60, population+refugees+male+age5_11, #population+refugees+male+age>60\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+refugees+female+age>60, population+refugees+male+age5_11, #population+refugees+male+age>60\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:27 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#population+refugees+female+age>60, population+refugees+male+age5_11, #population+refugees+male+age>60\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #population+refugees+female+age>60, population+refugees+male+age5_11, #population+refugees+male+age>60\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource a1a8ac08-807d-4bfb-9d97-2415134f21f1, ocha_nigeria_ne_cash_activities_May-Nov 2017-distributed.xlsx\n",
            "    Accessing data for resource c86ff8da-bf4b-4766-a43d-08f5d1fc5a15, Refugee Population in Afghanistan 2017\n",
            "    Accessing data for resource 4d9585ae-f32b-4742-a399-06efe44b63e8, Personnes déplacées du Pool\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:38 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+ secondary+school+f\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+ secondary+school+f\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:56:38 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#indicator+ secondary+school+f\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #indicator+ secondary+school+f\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource e6d5a229-92d2-4f76-9a0b-2f26854e1012, Iraq 2015.csv\n",
            "    Accessing data for resource d22dd1b6-2ff0-47ab-85c6-08aeb911a832, #HXL core vocabulary list\n",
            "    Accessing data for resource a1246506-28f6-4a41-b6f3-80ebdce74556, Standby Task Force Situational Review of Aid Responders in Nepal - Final 2W Report on 513 Organizations Responding as of May 6, 2015  - Final Output (1).xlsx\n",
            "    Accessing data for resource 2180251e-53dc-4ade-839d-2e390c0d406c, Mt_Agung\n",
            "Processing 590 of 612 (96.41%) resources\n",
            "    Accessing data for resource c7fb99a5-43ec-4b3f-b8db-935640c75aeb, assesment_data_crm_05april2017.xlsx\n",
            "    Accessing data for resource 8df22107-ffd4-4c06-a701-16ff5d1bd2b4, Afghanistan_Conflict_Displacements_2016.xlsx\n",
            "    Accessing data for resource f76b0dd8-2137-42a1-8a9b-4fc6dc38c007, Philippines 3w as of December 2016.xlsx\n",
            "    Accessing data for resource 98189d1d-7b20-4a5f-891e-17b26ef49a75, DC_OP4_DANA.xlsx\n",
            "    Accessing data for resource 2527ac5b-66fe-46f0-8b9b-7086d2c4ddd3, HNO -2017 -Sahel-nutrition.csv\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:57:12 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sector?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sector?\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:57:12 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sector?\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sector?\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 3106b05f-e171-49ac-a353-d7b56aed918f, Lake_Chad_Basin_Appeal_Status_2017-02-27.csv\n",
            "    Accessing data for resource 1aaa5281-de4f-4ffa-a5ef-37c754a86478, Lake_Chad_Basin_Displaced_2017-02-23.csv\n",
            "    Accessing data for resource 020000e5-e7aa-4027-b58a-efe7396ff32e, Lake_Chad_Basin_Estimated_Population_2017-02-23.csv\n",
            "    Accessing data for resource 48eea6e6-04d9-4c6a-ad38-80ff8365f601, 201612---Pakistan-4Ws-KP-FATA-2016-HXL (3).xlsx\n",
            "    Accessing data for resource 218ab806-0651-451a-a731-e0ccb80fe1d9, 06_09_update zika_global cases_IFRC.xlsx\n",
            "Processing 600 of 612 (98.04%) resources\n",
            "    Accessing data for resource 968202f1-856a-4906-ae87-c730e9b1dd27, PHL_haima_houses_damaged_pcoded_ndrrmc_sitrep_9_20161025.csv\n",
            "    Accessing data for resource 7211bf25-c9a1-4496-9bbb-a3e10efb2e4f, 15_09_zika_cases_export_Zika_Team.xlsx\n",
            "    Accessing data for resource 920df0fe-411d-47ee-8004-11b2302fbe1c, 160909_5W 2.0_HDX.xlsx\n",
            "    Accessing data for resource ac8d491f-338f-4141-8f77-8480e6f6c414, 160906_5W_HDX.xlsx\n",
            "    Accessing data for resource 12e99d1f-8dc7-422a-9336-91c515e3c866, 160614_ECUADOR_5W_HDX.xlsx\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:57:54 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sector#subsector\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sector#subsector\n",
            "ERROR:hxl.REMOTE_ACCESS:2024-07-31 15:57:54 [error    ] Skipping column(s) with malformed hashtag specs function=parse_list hastags=#sector#subsector\n",
            "ERROR:hxl.model:Skipping column(s) with malformed hashtag specs: #sector#subsector\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Accessing data for resource 58069fa8-18b6-4132-a242-32216815a398, Ecuador Earthquake - April 2016 - Severity index\n",
            "    Accessing data for resource 5c3a03d2-53de-4754-86b3-c7116d927155, Burundi 3W_formated.xlsx\n",
            "    Accessing data for resource 62d4b11f-1a29-4753-a54f-c327f21f0b0b, 3W - 5 May - Final consolidated.xlsx\n",
            "    Accessing data for resource f7daccff-bcc7-41cd-bee0-4133ce6b8988, Somalia NGO Consortium Data.xlsx\n",
            "    Accessing data for resource 9cba28b4-d112-4850-96b4-81122af45f9b, 141121 LR Health Care Facilities.xlsx\n",
            "Processing 610 of 612 (99.67%) resources\n",
            "    Accessing data for resource f78dc606-04e2-4fb6-a7eb-9eb995c33f76, 1501 Sierra Leone Health Centers.xlsx\n",
            "    Accessing data for resource 5d2531d6-c03a-449b-afdd-52c07d687679, Guinea health-facility master data\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "repr_error": "0",
              "type": "dataframe",
              "variable_name": "df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-b389eccf-4cea-4664-a8a8-1cdb27c2719a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "      <th>Text header</th>\n",
              "      <th>Locations</th>\n",
              "      <th>Data provider</th>\n",
              "      <th>HDX dataset id</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Date created</th>\n",
              "      <th>Unnamed: 9</th>\n",
              "      <th>Hash</th>\n",
              "      <th>Data excerpt</th>\n",
              "      <th>File</th>\n",
              "      <th>URL</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>#affected+hh</td>\n",
              "      <td>Total IDP HH</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[319283]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>#affected+ind+returnees</td>\n",
              "      <td>Total Returnees</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[587705]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>#country</td>\n",
              "      <td>country_name</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101</th>\n",
              "      <td>#adm1+name</td>\n",
              "      <td>region_name</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td></td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>102</th>\n",
              "      <td>#geo+lat</td>\n",
              "      <td>latitude</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689531</th>\n",
              "      <td>#adm1</td>\n",
              "      <td>Province</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689533</th>\n",
              "      <td>#adm2</td>\n",
              "      <td>District</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689535</th>\n",
              "      <td>#adm3</td>\n",
              "      <td>Chiefdom</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689545</th>\n",
              "      <td>#adm1+name</td>\n",
              "      <td>Nom de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689546</th>\n",
              "      <td>#adm1+code</td>\n",
              "      <td>P-code de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3777 rows × 12 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b389eccf-4cea-4664-a8a8-1cdb27c2719a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b389eccf-4cea-4664-a8a8-1cdb27c2719a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b389eccf-4cea-4664-a8a8-1cdb27c2719a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f160952a-868d-4c82-899b-8df77ff5ed85\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f160952a-868d-4c82-899b-8df77ff5ed85')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f160952a-868d-4c82-899b-8df77ff5ed85 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_994c2ec9-dfa4-4324-b8af-60fcfbf44cdf\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_994c2ec9-dfa4-4324-b8af-60fcfbf44cdf button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "        Hashtag with Attributes          Text header  \\\n",
              "1                  #affected+hh         Total IDP HH   \n",
              "8       #affected+ind+returnees      Total Returnees   \n",
              "100                    #country         country_name   \n",
              "101                  #adm1+name          region_name   \n",
              "102                    #geo+lat             latitude   \n",
              "...                         ...                  ...   \n",
              "689531                    #adm1             Province   \n",
              "689533                    #adm2             District   \n",
              "689535                    #adm3             Chiefdom   \n",
              "689545               #adm1+name     Nom de la région   \n",
              "689546               #adm1+code  P-code de la région   \n",
              "\n",
              "                                                                                          Locations  \\\n",
              "1                                                                                               COD   \n",
              "8                                                                                               COD   \n",
              "100     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "101     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "102     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "...                                                                                             ...   \n",
              "689531                                                                                          SLE   \n",
              "689533                                                                                          SLE   \n",
              "689535                                                                                          SLE   \n",
              "689545                                                                                          GIN   \n",
              "689546                                                                                          GIN   \n",
              "\n",
              "                                   Data provider  \\\n",
              "1       international-organization-for-migration   \n",
              "8       international-organization-for-migration   \n",
              "100         eth-zurich-weather-and-climate-risks   \n",
              "101         eth-zurich-weather-and-climate-risks   \n",
              "102         eth-zurich-weather-and-climate-risks   \n",
              "...                                          ...   \n",
              "689531                        standby-task-force   \n",
              "689533                        standby-task-force   \n",
              "689535                        standby-task-force   \n",
              "689545                        ipc-cluster-guinea   \n",
              "689546                        ipc-cluster-guinea   \n",
              "\n",
              "                                                                                    HDX dataset id  \\\n",
              "1       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "8       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "100                                                                     climada-earthquake-dataset   \n",
              "101                                                                     climada-earthquake-dataset   \n",
              "102                                                                     climada-earthquake-dataset   \n",
              "...                                                                                            ...   \n",
              "689531                                                       141121-sierra-leone-health-facilities   \n",
              "689533                                                       141121-sierra-leone-health-facilities   \n",
              "689535                                                       141121-sierra-leone-health-facilities   \n",
              "689545                                                               guinea-healthcare-master-data   \n",
              "689546                                                               guinea-healthcare-master-data   \n",
              "\n",
              "                             HDX resource id Date created Unnamed: 9  \\\n",
              "1       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "8       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "100     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "101     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "102     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "...                                      ...          ...        ...   \n",
              "689531  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689533  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689535  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689545  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "689546  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "\n",
              "                      Hash  \\\n",
              "1       0x2cc7fd3129c0d18c   \n",
              "8       0x2cc7fd3129c0d18c   \n",
              "100     0x234606b299c1e43e   \n",
              "101     0x234606b299c1e43e   \n",
              "102     0x234606b299c1e43e   \n",
              "...                    ...   \n",
              "689531  0x5c648c638f567754   \n",
              "689533  0x5c648c638f567754   \n",
              "689535  0x5c648c638f567754   \n",
              "689545  0x47c11098111f7d90   \n",
              "689546  0x47c11098111f7d90   \n",
              "\n",
              "                                                                                                                                                                 Data excerpt  \\\n",
              "1                                                                                                                                                                    [319283]   \n",
              "8                                                                                                                                                                    [587705]   \n",
              "100     ['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']   \n",
              "101                                                                                                                                                                             \n",
              "102                                                 ['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']   \n",
              "...                                                                                                                                                                       ...   \n",
              "689531                                              ['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']   \n",
              "689533                                                         ['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']   \n",
              "689535                                                                               ['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']   \n",
              "689545                                                                                       ['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']   \n",
              "689546                                                                                       ['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']   \n",
              "\n",
              "                                                                                                                             File  \\\n",
              "1       /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "8       /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "100                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "101                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "102                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "...                                                                                                                           ...   \n",
              "689531                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689533                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689535                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689545                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "689546                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "\n",
              "                                                                                                                                                                        URL  \n",
              "1       https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx  \n",
              "8       https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx  \n",
              "100            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv  \n",
              "101            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv  \n",
              "102            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv  \n",
              "...                                                                                                                                                                     ...  \n",
              "689531   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx  \n",
              "689533   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx  \n",
              "689535   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx  \n",
              "689545                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0  \n",
              "689546                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0  \n",
              "\n",
              "[3777 rows x 12 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(3777, 9)\n",
            "(3777, 12)\n"
          ]
        }
      ],
      "source": [
        "def populate_data_excerpts(df_in):\n",
        "    \"\"\"\n",
        "    Populates the 'Data excerpt' column in the given DataFrame with data excerpts from each resource's columns.\n",
        "\n",
        "    Args:\n",
        "        df_in (pandas.DataFrame): The input DataFrame containing the data.\n",
        "\n",
        "    Returns:\n",
        "        pandas.DataFrame: The modified DataFrame with the 'Data excerpt' column populated.\n",
        "    \"\"\"\n",
        "\n",
        "    df = df_in.copy()\n",
        "\n",
        "    df[\"Data excerpt\"] = \"\"\n",
        "\n",
        "    datasets_resources = df[[\"HDX dataset id\", \"HDX resource id\"]].drop_duplicates()\n",
        "    datasets_resources.reset_index(drop=True, inplace=True)\n",
        "\n",
        "    print(\"For each resource, extract a data excerpt for each column ...\")\n",
        "\n",
        "    num_rows = datasets_resources.shape[0]\n",
        "    for index, row in datasets_resources.iterrows():\n",
        "\n",
        "        if index % 10 == 0:\n",
        "            print(\n",
        "                f\"Processing {index} of {num_rows} ({(index/num_rows)*100:.2f}%) resources\"\n",
        "            )\n",
        "\n",
        "        dataset_id = row[\"HDX dataset id\"]\n",
        "        resource_id = row[\"HDX resource id\"]\n",
        "        try:\n",
        "            dataset = Dataset.read_from_hdx(dataset_id)\n",
        "            if dataset is None:\n",
        "                print(f\"Dataset {dataset_id} not found!\")\n",
        "                continue\n",
        "        except Exception as e:\n",
        "            print(f\"Error reading dataset {dataset_id} ... {e}\")\n",
        "            continue\n",
        "\n",
        "        resources = dataset.get_resources()\n",
        "        for resource in resources:\n",
        "            if resource[\"id\"] == resource_id:\n",
        "                print(\n",
        "                    f\"    Accessing data for resource {resource_id}, {resource['name']}\"\n",
        "                )\n",
        "                try:\n",
        "                    url, path = resource.download(LOCAL_DATA_DIR)\n",
        "                    df.loc[df[\"HDX resource id\"] == resource_id, \"File\"] = path\n",
        "                    df.loc[df[\"HDX resource id\"] == resource_id, \"URL\"] = url\n",
        "\n",
        "                    with hxl.data(resource[\"url\"], input_options) as source:\n",
        "                        columns = [column.header for column in source.columns]\n",
        "                        tags = [\n",
        "                            column.get_display_tag(sort_attributes=True)\n",
        "                            for column in source.columns\n",
        "                        ]\n",
        "                        data = {}\n",
        "                        rowcount = 0\n",
        "                        for row in source:\n",
        "                            if rowcount > DATA_EXCERPT_SIZE:\n",
        "                                break\n",
        "                            i = 0\n",
        "                            for colvalue in row:\n",
        "                                colname = columns[i]\n",
        "                                if colname not in data:\n",
        "                                    data[colname] = [colvalue]\n",
        "                                else:\n",
        "                                    data[colname].append(colvalue)\n",
        "                                i += 1\n",
        "                            rowcount += 1\n",
        "\n",
        "                        for col in columns:\n",
        "                            if col in data:\n",
        "                                # print(f\"       Setting data excerpt for column {col} >> {data[col]} ...\")\n",
        "                                df.loc[\n",
        "                                    (df[\"HDX resource id\"] == resource_id)\n",
        "                                    & (df[\"Text header\"] == col),\n",
        "                                    \"Data excerpt\",\n",
        "                                ] = str(data[col])\n",
        "\n",
        "                except Exception as e:\n",
        "                    print(\n",
        "                        f\"Error accessing data for resource {resource_id}, {resource['name']} ... {e}\"\n",
        "                    )\n",
        "\n",
        "    return df\n",
        "\n",
        "\n",
        "df = data.copy()\n",
        "df = populate_data_excerpts(df)\n",
        "display(df)\n",
        "\n",
        "df.to_csv(f\"{LOCAL_DATA_DIR}/hxl_hash_resources_data.csv\", index=False)\n",
        "\n",
        "print(data.shape)\n",
        "print(df.shape)\n",
        "\n",
        "data = df.copy()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3HFnUUNxWgAG"
      },
      "source": [
        "## Data cleaning\n",
        "\n",
        "Let's remove rows where Data excerpts weren't found. This may be due to blank tables, processing errors and more."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0nAMfOhNWl0c",
        "outputId": "f6350042-0faa-4aa1-ed35-305050ffd16f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(3341, 12)\n"
          ]
        }
      ],
      "source": [
        "data = data[data[\"Data excerpt\"].notnull()]\n",
        "data = data[data[\"Data excerpt\"].str.contains(r\"[A-Za-z0-9]\")]\n",
        "\n",
        "print(data.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rWjayNp_WvbU"
      },
      "source": [
        "## Tags distribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7j7WxJKGWyny",
        "outputId": "0d4fb38c-5bae-4d5a-e4b1-0d8a9e40e93b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tag\n",
            "#adm1           523\n",
            "#adm2           473\n",
            "#affected       380\n",
            "#country        284\n",
            "#date           244\n",
            "#org            228\n",
            "#adm3           188\n",
            "#inneed         140\n",
            "#sector         109\n",
            "#geo            106\n",
            "#targeted        75\n",
            "#loc             67\n",
            "#activity        67\n",
            "#status          59\n",
            "#population      57\n",
            "#indicator       55\n",
            "#region          49\n",
            "#meta            42\n",
            "#reached         32\n",
            "#adm4            32\n",
            "#subsector       13\n",
            "#event           13\n",
            "#beneficiary     12\n",
            "#cause           12\n",
            "#value           11\n",
            "#item            10\n",
            "#severity         9\n",
            "#output           8\n",
            "#crisis           6\n",
            "#currency         4\n",
            "#service          4\n",
            "#adm5             4\n",
            "#contact          4\n",
            "#access           4\n",
            "#capacity         3\n",
            "#impact           3\n",
            "#description      3\n",
            "#frequency        2\n",
            "#group            2\n",
            "#modality         2\n",
            "#delivery         1\n",
            "#operations       1\n",
            "Name: count, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "data[\"tag\"] = data[\"Hashtag with Attributes\"].apply(lambda x: x.split(\"+\")[0])\n",
        "tag_counts_train = data[\"tag\"].value_counts()\n",
        "\n",
        "print(tag_counts_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UElfy2V7deQH"
      },
      "source": [
        "## Generate Data table descriptions using LLM\n",
        "Though there are descriptions on HDX for datasets, there aren't always detailed descriptions for tables in CSV and Excel files. We will generate these using GPT-3.5-Turbo to include in the prompt as they provide valuable context when considering what HXL tags apply to a column.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "hNdIGK1Id1IC",
        "outputId": "75c89bb3-9735-4946-b1bb-7cb6b9ca3d37"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Unique resources: 555\n",
            "\n",
            "Description: The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Democratic Republic of Congo. The data includes the total number of IDP households, individual IDPs, male and female IDPs, and returnees. For instance, there are 319,283 IDP households, 1,548,732 individual IDPs, 646,805 male IDPs, 901,927 female IDPs, and 587,705 returnees.\n",
            "Description: The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.\n",
            "Description: The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.\n",
            "Description: The dataset contains information related to funding requirements and funding received for activities in Palestine (PSE). It includes details such as country code, activity ID, name, start and end dates, year, funding requirements in USD, total funding received in USD, and the percentage of funding received. The data shows various activities with specified funding details, as well as unspecified activities with missing information. The dataset spans multiple years, with different activities and funding amounts recorded for each year.\n",
            "Description: The dataset contains information on incoming funding transactions, including details such as date, budget year, description, amount in USD, source organization, destination organization, contribution type, flow type, method, and status. The data includes funding from various governments, organizations, and multilateral entities for projects in different regions, with a focus on sectors like protection, mine action, and counter-terrorism. The dataset also captures details on project codes, emergency designations, usage years, and financial aspects like original amounts, currencies, exchange rates, and financial directions.\n",
            "Description: The dataset contains information on donor contributions, including details such as activity date, contribution code, contribution ID, country code, donor information, commitment, pledge, received amount, write-off, latest date, donor type, region name, status code, and year. The data includes contributions from various countries and regions, with corresponding financial amounts and dates.\n",
            "Description: The dataset contains information on security incidents in Sudan, including incident ID, year, month, day, country code, country, region, district, city, organizations affected, casualties (nationals and internationals killed, wounded, kidnapped), means of attack, attack context, location details, latitude, longitude, motive, actor type, actor name, and additional details. The incidents range from aerial bombardments to ambushes and involve various organizations and individuals being targeted or affected. The dataset appears to be structured with columns representing different attributes of each incident, providing a detailed overview of security incidents in Sudan over multiple years.\n",
            "Description: The dataset contains information on various events related to insecurity in Sudan from 2016 to 2023, including data on Aid Worker KIKA Events, Attacks on Health Care Events, Conflict-related Sexual Violence Events, Education in Danger Events, Explosive Weapons Events, Food Security Protection in Danger Events. The data shows the number of events recorded each year for these categories, with some missing values for certain years. The dataset provides insights into the trends and occurrences of insecurity events in Sudan over the specified time period.\n",
            "Description: The dataset is from DTM Sudan and provides information on displacement and return trends in Sudan. It includes a disclaimer about the challenges faced in data collection due to military clashes, limited humanitarian access, and connectivity issues. The data is collected through remote interviews with key informants and is subject to verification. The dataset also includes instructions on how to read it and a citation requirement for using the data.\n",
            "Description: The dataset contains information on floods in various regions of Sudan, including details such as state, locality, village name, houses destroyed, houses damaged, households affected, and people affected. The data shows the impact of floods in different areas, with varying levels of damage and displacement. The dataset provides insights into the extent of destruction caused by floods in different regions, highlighting the need for disaster response and mitigation efforts to support affected communities.\n",
            "Processed 2.0% resources ...\n",
            "Description: The dataset contains information on population types, origins, and asylum statuses in Nigeria, with details on administrative divisions and accommodation types. It includes demographic breakdowns by age and gender, as well as total population figures. The data seems to be sourced from UNHCR and government sources, with a focus on registration statistics.\n",
            "Description: The dataset contains information on various indicators related to health and demographics in Sao Tome and Principe, with data points for different regions within the country. The indicators include total fertility rate for women aged 15-49, percentage of married women currently using contraception, and percentage of married women using modern contraception methods. The data includes values, precision, survey details, and characteristic categories for each indicator. The dataset covers the year 2008 and is sourced from the DHS (Demographic and Health Surveys) program.\n",
            "Description: The dataset contains various indicators related to demographic and health statistics for Sao Tome and Principe in 2008, sourced from the DHS Quickstats Data file. The indicators include total fertility rate, contraceptive use among married women, unmet need for family planning, median age at first marriage and sexual intercourse, and infant mortality rates. Each data entry includes information such as the indicator name, value, precision, country code, survey details, and characteristic categories. The dataset provides insights into key health and demographic trends in Sao Tome and Principe during the specified survey year.\n",
            "Description: The dataset contains information on WFP food prices for various countries, including country ISO codes, URLs linking to the data, start and end dates of the data collection. The data spans different time periods for each country, with some data going back to the 1990s and others extending into the 2020s. The dataset seems to be structured with columns for country ISO codes, URLs, start dates, and end dates, providing a comprehensive overview of WFP food price data for multiple countries.\n",
            "Description: The dataset contains information on food prices in various markets in Zimbabwe, with details such as date, location (admin1 and admin2), market name, geographical coordinates, category, commodity, unit, price flag, price type, currency, price in local currency, and price in USD. The data spans from 2010-01-15 and includes markets like Kombai, Mandava, Mucheke, Murombedzi, Renkini Bus Terminus, Tshovani, Mbare, Nyanyadzi, and Sakubva. The prices are primarily for maize sold in kilograms, with prices ranging from 0.23 to 0.43 USD per kilogram.\n",
            "Description: The dataset contains information on various regions and organizations involved in humanitarian activities in the Central African Republic. It includes details such as the prefecture, sub-prefecture, sector, actor, organization type, description of activities/projects, start and end dates, status, update date, zone, and data source. The activities range from constructing new wells to distributing hygiene kits and rehabilitating existing wells. Multiple entries exist for some regions and organizations, indicating repeated activities. The dataset seems to focus on humanitarian efforts related to water and sanitation projects in different regions of the country.\n",
            "Description: The dataset contains information on various administrative regions in the Central African Republic, including the names and codes of different administrative levels. It also includes the number of sites, families accommodated, and total figures for each region for a specific month. The data covers regions such as Bangui, Nana-Mambéré, and Basse-Kotto, with details on the number of families accommodated in each area. Additionally, there are missing values in some columns, and the dataset seems to be focused on tracking accommodation and support services provided in these regions.\n",
            "Description: The dataset contains information on CERF (Central Emergency Response Fund) allocations, including details such as agency name, continent, country code, country name, date of USG signature, emergency type, project code, project ID, project title, region name, total amount approved, window name, year, project sectors, clusters, and groupings. The data appears to focus on emergency response projects related to drought mitigation in various African countries, with specific allocations made by the Food and Agriculture Organization for immediate support to pastoral communities. The dataset includes multiple entries for different countries, each detailing the allocated amount and project specifics.\n",
            "Description: The dataset contains information on displacement tracking in Yemen from 11 to 17 February 2024. It includes data on the country, country code, reported date, governorate of displacement, IDP households displaced during the week, and returnee households displaced during the week. The data shows the number of affected households in different governorates, with some entries having missing values.\n",
            "Description: The dataset contains information on Internally Displaced Persons (IDPs) in Lebanon, with columns for Governorate, District, Cadaster, number of households (HHs), number of IDPs, and various other related data points. The data includes details such as the Governorate and District names and codes, as well as information on returnees, rented houses, hosted individuals, and collective shelters. The dataset provides insights into the displacement situation in Lebanon, including the movement of IDPs between different areas and the support structures in place for them.\n",
            "Processed 4.0% resources ...\n",
            "Description: The dataset contains information on disasters that occurred in various countries in the year 2000. It includes details such as the country, disaster type, total events, total affected individuals, total deaths, total damage in USD, and the Corruption Perceptions Index (CPI) for each country. The disasters recorded include droughts, floods, storms, wildfires, and landslides. The dataset provides a snapshot of the impact of natural disasters on different countries in the year 2000, highlighting the number of affected individuals, deaths, and economic damage caused by these events.\n",
            "Description: The dataset contains information related to food security indicators for Zimbabwe, with columns including Iso3 (country code), StartDate, EndDate, Area Code, Area, Item Code, Item, Element Code, Element, Year Code, Year, Unit, Value, Flag, and Note. The data includes details such as the average dietary energy supply adequacy percentage over a 3-year average period for Zimbabwe from 2000 to 2010. The values range from 95% to 101% during this time frame.\n",
            "Description: The dataset contains information on GDP deflator values for Zimbabwe from 1970 to 1978. Each entry includes details such as the ISO3 country code, start and end dates, area code, item code, element code, year, unit, and the GDP deflator value in standard local currency at 2015 prices. The data is structured in a tabular format with columns representing different attributes of the data entries.\n",
            "Description: The dataset contains exchange rate data for the Holy See in 1999, with information on the local currency units per USD for each month. The data includes columns such as ISO code, start and end dates, area code, element code, currency information, year, month, unit, value, and flag. The exchange rates are provided for each month of the year 1999, showing the conversion rate of the Euro to local currency units per USD.\n",
            "Description: The dataset contains information on water points in Haiti, including latitude, longitude, status, report date, water source, water technology, country details, administrative divisions, installation and rehabilitation details, management, payment status, water quality indicators, population data, facility type, distance to various locations, urban status, days since report, staleness, prediction outcomes, and predicted categories. The data includes details on various water points such as protected wells and piped water tapstands, their functionality status, and repair priorities.\n",
            "Description: The dataset contains information on the country, total population, validity period, analysis date, severity phase, number of affected individuals, and percentage of affected population. The data includes details for Afghanistan for March 2024, with different severity phases and projected periods. The total country population is 44,532,600, and the dataset provides breakdowns of affected populations for different severity phases and projected periods.\n",
            "Description: The dataset from the file \"Mozambique Crisis - MT Round 20 (Dec 2023)3.xlsx\" contains information on the total number of men, women, and children in Cabo Delgado, with a grand total provided as well. The data is structured in rows and columns, with specific labels for each category. For instance, the total number of men, women, and children are listed under the respective columns. The dataset also includes subtotals for each category within Cabo Delgado.\n",
            "Description: The dataset contains information on activities and projects related to member organizations in Zimbabwe, including details such as titles, dates, summaries, content, topics, working groups, sectors, funding types, and other tags. The data includes examples of initiatives aimed at addressing issues such as child malnutrition and maternal and newborn health through technology and partnerships with organizations like UNICEF and Promobile. The dataset also includes information on the authors, status, and country codes associated with each entry.\n",
            "Description: The dataset contains information on new admissions in healthcare facilities in different regions and districts. It includes columns for the region, district health facility, total new admissions for different categories (CRENAS, CRENI, CRENAM), and the total number of individuals present at the beginning of the year. The data shows varying numbers of new admissions across different regions and districts, with some districts having higher numbers of new admissions compared to others.\n",
            "Description: The dataset contains information on operational presence in Somalia, including clusters, organization names, acronyms, types, regions, and districts. It appears to list various organizations involved in different regions and districts within Somalia, such as international NGOs (INGOs) and national NGOs (NNGOs). The data includes details like the cluster, organization name, acronym, organization type, region, and district for each entry.\n",
            "Processed 5.0% resources ...\n",
            "Description: The dataset contains information on various sectors, states, localities, and organizations involved in humanitarian response activities. It includes data for the reporting month of December, with details such as sector type, state names, locality names, whether the organization is part of the Humanitarian Response Plan (HRP) or outside of it, organization names, acronyms, and types (e.g., INGO, NNGO). The data appears to be structured in a tabular format with columns representing different attributes of the organizations and their activities in different regions.\n",
            "Description: The dataset contains daily exchange rate data for various currencies against the USD, including EUR, JPY, GBP, CAD, and more. Each row represents a specific date, and the columns include the date, exchange rates for different currencies, such as BGN, CZK, DKK, HUF, PLN, SEK, CHF, and others. The data spans multiple dates, with exchange rate values provided for each currency on those dates.\n",
            "Description: The dataset contains Human Development Indicators for Zimbabwe, with information on indicators such as Adolescent Birth Rate (births per 1,000 women ages 15-19) for the years 1990 to 1998. The data includes columns for country code, country name, indicator ID, indicator name, index ID, index name, indicator value, and year. The values in the dataset show a trend of decreasing Adolescent Birth Rate over the specified years.\n",
            "Description: The dataset contains Human Development Indicators for Somalia, with information on indicators such as Adolescent Birth Rate (births per 1,000 women ages 15-19) for the years 1990 to 1998. The data includes columns for country code, country name, indicator ID, indicator name, index ID, index name, indicator value, and year. The values in the dataset show a trend of Adolescent Birth Rate fluctuating over the years, ranging from 115.367 to 126.275.\n",
            "Description: The dataset contains demographic and humanitarian condition information for different districts in Yemen. It includes data on total population, PiN (People in Need) disaggregated by gender and various sectors such as RMMS, WASH, Shelter, Nutrition, Education, FSAC, CCCM, Health, RRM, Protection, GBV, Mine Action, Child Protection, and more. Each district is categorized based on severity levels ranging from Minimal to Extreme, with corresponding figures for different demographic groups and sectors. The dataset provides a detailed overview of the humanitarian situation in these districts, highlighting the needs and conditions of the population.\n",
            "Description: The dataset contains information on natural disaster incidents in Afghanistan from January to November 2023, recorded by OCHA Field Offices and partners. It includes incidents such as avalanches, earthquakes, flash floods, floods, heavy rainfall/snowfall, landslides, and mudflows. The data has been aggregated by district to unique events based on the type of natural disaster and date. The incidents have impacted Afghans, potentially requiring humanitarian assistance. The dataset combines information from HADB and OCHA Field Office reports, including assessment figures from various organizations.\n",
            "Description: The dataset contains information on conflicts in Zimbabwe, including details such as conflict ID, year, type of violence, conflict name, parties involved, location, sources, and casualties. Each entry includes specific data points such as the date, location coordinates, description, administrative divisions, and number of deaths among different groups involved. The dataset covers conflicts from 1995 to 2008, involving the Government of Zimbabwe and civilians, with sources ranging from news articles to official reports. The data provides a comprehensive overview of various conflict incidents in Zimbabwe during the specified time period.\n",
            "Description: The dataset is a Mobility Tracking (MT) matrix from the Displacement Tracking Matrix (DTM) in the Lac Province of Chad, implemented by the International Organization for Migration (IOM) since May 2015 to monitor population movements due to attacks by non-state armed groups. The data includes project summaries, objectives, methodology, limitations, target populations (internally displaced persons, returnees, and those returning from abroad), data collection periods, geographic coverage across 281 localities in 7 sub-prefectures, and accessibility information. The data is collected through surveys conducted with key informants and analyzed to provide insights for humanitarian and development strategies.\n",
            "Description: The dataset contains information related to emergency events in Burundi, including details such as the reported date, incident date, province, commune, type of emergency, damages to houses, number of houses affected, other damages to infrastructure, number of affected households and individuals, number of displaced households and individuals, urgent needs related to the incident, humanitarian assistance provided, and any additional comments. The data includes specific details for each event, such as the type and extent of damages, number of affected individuals, and types of urgent needs and assistance provided.\n",
            "Description: The dataset contains information about airports in Myanmar, including details such as airport ID, code, type, name, geographical coordinates, elevation, continent, country, region, municipality, scheduled service status, GPS code, IATA code, URLs, keywords, score, and last updated date. It includes large, medium, and small airports in various regions of Myanmar, with details such as Yangon International Airport, Mandalay International Airport, Bagan Airport, Sittwe Airport, Thandwe Airport, Nay Pyi Taw International Airport, Dawei Airport, Myeik Airport, and Kengtung Airport.\n",
            "Processed 7.000000000000001% resources ...\n",
            "Description: The dataset contains information on various humanitarian response plans and hazard data for disaster risk assessment from different sources and organizations. It includes details such as dataset names, titles, organizations, sources, and creation/update dates. The data covers topics like displacement trends tracking, storm, wildfire, flood, earthquake, crop production, operational presence in Mozambique, attacks on protection in Bangladesh, and exposure data for disaster risk assessment. The dataset appears to be focused on providing valuable information for humanitarian efforts and disaster management.\n",
            "Description: The dataset contains information for Zimbabwe from 1989 to 1997 on the number of agricultural machinery tractors, with corresponding values for each year. The data includes columns for Country Name, Country ISO3 code, Year, Indicator Name, Indicator Code, and the numerical Value. The dataset appears to be structured with consistent data format and can be used for analyzing trends in agricultural machinery in Zimbabwe over the specified time period.\n",
            "Description: The dataset contains information on displaced persons in UNRWA shelters by governorate, with columns including Date, Territory, Governorate, Cumulative Displaced Persons in UNRWA Shelters, Daily Displaced Persons in UNRWA Shelters, Cumulative UNRWA Shelters, and Daily UNRWA Shelters. The data covers multiple dates and governorates within the Gaza Strip region, detailing the number of displaced persons and shelter capacities on a daily basis. The dataset also includes columns for Territory p-code, Governorate p-code, and an Uncertain column.\n",
            "Description: The dataset contains population statistics for various administrative regions in Haiti, including total population, number of females, males, children, adults, girls, boys, adult females, and adult males. Each row represents a different administrative region, with corresponding data for each category. The dataset includes information such as the ISO3 country code, administrative region codes and names, and population breakdowns by gender and age group.\n",
            "Description: The dataset provided in the file UKR_3W_2021_Q33.xlsx contains information on Protection and Shelter Cluster partners operating in Non-government controlled areas of Donetska and Luhanska oblasts in Ukraine. The data includes details such as the organization name, abbreviation, type, administrative region name, and administrative region code. The organizations listed in the dataset are involved in various sectors, with a focus on education. The dataset includes both national and international non-governmental organizations (NGOs) working in the specified regions.\n",
            "Description: The dataset contains information from a baseline assessment conducted in various departments and communes in Haiti in December 2023. The data includes details such as the evaluation round date, enumerator's gender, data collection method, site type, site characteristics, site management information, presence of internally displaced persons (IDPs) and returnees, demographics of individuals at the sites, reasons for displacement, return, and needs assessment categories. The dataset also includes information on key informants interviewed in groups. The data provides insights into the population, living conditions, and displacement dynamics in the specified areas.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (0,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on demographics and locations of forcibly displaced and stateless persons globally. It includes data on the year, country of origin and asylum, population type, location, urban/rural classification, accommodation type, and demographic breakdown by age and gender. The data provides details on the number of affected individuals in different categories such as infants, children, adolescents, adults, and elderly, as well as unknown age groups. The dataset covers various countries and years, with total population figures included.\n",
            "Description: The dataset contains information on projects in Ethiopia for December 2023, including details such as reporting month, project status, donor organizations, implementing agencies, locations at regional, zonal, and woreda levels, sector clusters, activities, and geographical coordinates. The data shows ongoing projects primarily focused on community participation and self-governance in various locations within the Tigray region. The implementing agency is the UN Agency IOM, with multiple projects in Mekelle and Adigrat town. The dataset provides latitude and longitude coordinates for each project location.\n",
            "Description: The dataset contains information on various regions in Ukraine, including data on targets and activities related to different clusters such as IDPs, non-displaced individuals, and returnees. The data includes columns for Oblast, overall targets, and specific targets for different clusters within each region. The dataset also includes columns for different types of activities and targets related to displacement and returnees. The dataset provides detailed information on targets and activities for each region in Ukraine.\n",
            "Description: The dataset contains information on different administrative regions and villages in Cameroon, specifically focusing on the number of households and individuals displaced due to various reasons such as conflict or natural disasters. The data includes columns for administrative region names and codes, village names and codes, as well as counts for household IDPs (Internally Displaced Persons), individual IDPs, household refugees, individual refugees, household returnees, individual returnees, and the total number of displaced individuals. The dataset provides a detailed breakdown of displacement figures for each village within the specified administrative regions.\n",
            "Processed 9.0% resources ...\n",
            "Description: The dataset contains information on the number of internally displaced persons (IDPs) and returnees in different regions of Mali. The data includes columns for the regions, total IDPs, returnee IDPs, returnees from abroad, and total returnees. The regions listed include Bamako, Bandiagara, Bougouni, Dioila, Douentza, Gao, Kayes, Kidal, and Kita. The total number of IDPs and returnees varies across the regions, with some regions having significant numbers of returnees from abroad. The dataset provides a snapshot of the displacement and returnee situation in Mali's different regions.\n",
            "Description: The dataset contains information on different regions in Ukraine, including country codes, oblast names in English and Ukrainian, raion names in English and Ukrainian, and the number of internally displaced persons (IDPs) recorded as arrivals in each raion. The data shows the top raions with the highest number of recorded IDP arrivals, with Kyiv City having the highest number at 290,279, followed by Kharkivskyi, Zaporizkyi, Dniprovskyi, Lvivskyi, Odeskyi, Buchanskyi, Poltavskyi, and Mykolaivskyi raions.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Afghanistan COVID-19 Stats3.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Afghanistan COVID-19 Stats3.google sheet\n",
            "Description: The dataset from the file HTI_HNO_20243.xlsx contains administrative data for Haiti, including columns for Admin 1 name and code, Admin 2 name and code, and PiN (People in Need) figures. The data includes information on various regions within the Ouest department, such as Port-au-Prince, Delmas, Carrefour, Pétion-Ville, Kenscoff, Gressier, Cité Soleil, and Tabarre, along with corresponding P-Codes and PiN values.\n",
            "Description: The dataset contains information on access classification for different localities in various states for September 2023. It includes columns such as State, PCODE_Admin1, Locality, PCODE_Admin2, and Access - September 2023. The access status for each locality is categorized as either \"Hard to reach\" or \"Accessible.\" The dataset provides details on the accessibility of different areas within the states, with some areas being classified as hard to reach while others are considered accessible.\n",
            "Description: The dataset contains information on different regions, health districts, and prefectures in the context of malnutrition in the Central African Republic. It includes columns such as Region Sanitaire, District Sanitaire, Prefecture, Sous-Prefecture, and various malnutrition indicators for children aged 6-59 months. Each row represents a specific location within the country and provides data on malnutrition prevalence rates.\n",
            "Description: The dataset contains information related to various humanitarian assistance projects in Colombia, including project details, organization information, activity descriptions, COVID-19 relevance, resource types, estimated costs, beneficiary demographics, and project statuses. Each entry includes data such as project names, codes, implementing organizations, activity descriptions, target populations, country details, and the number of individuals reached. The dataset also includes information on the types of activities being implemented, such as the delivery of aid kits, educational support, and protection services. Additionally, it captures details on the start and end dates of activities, as well as indicators related to COVID-19, natural disasters, and armed violence.\n",
            "Description: The dataset contains information on different administrative regions in Chad, including the state, county, and corresponding codes. It also includes data on the number of people in need (PIN) and the targeted population (Cible) for each region. The dataset is structured in a tabular format with columns such as ADM1_STATE, ADM1_Pcode, ADM2_COUNTY, ADM2_Pcode, PIN, and Cible. The data provides insights into the population in need and targeted for assistance in various regions of Chad.\n",
            "Description: The dataset contains information on acute malnutrition in Nigeria, specifically focusing on Severe Acute Malnutrition (SAM) and Moderate Acute Malnutrition (MAM) among children under 5 years old in different states and local government areas (LGAs). The data includes columns for State, LGA, SAM U5, and MAM U5, with values representing the prevalence of SAM and MAM in each region. The dataset provides detailed figures for each region, indicating the severity of acute malnutrition in different parts of Nigeria.\n",
            "Description: The dataset titled \"Myanmar 2024 HPC People in Need Breakdowns\" contains information on various regions in Myanmar, including data on education, food security, health, nutrition, protection, shelter, WASH, and overall people in need. The data includes projections for IDPs, returnees, stateless people, and other crisis-affected individuals, categorized by gender, age groups, and persons with disabilities. Each region is represented with corresponding figures for the different categories, providing a comprehensive overview of humanitarian needs in Myanmar for the year 2024.\n",
            "Description: The dataset contains information on different administrative regions (Admin 1 and Admin 2) and the corresponding sites within those regions, along with the estimated number of individuals as of 13/12. The data includes details such as the region names, site names, and the estimated number of affected individuals or returnees in each site. The dataset provides a snapshot of the population distribution across various sites within the specified administrative regions.\n",
            "Processed 11.0% resources ...\n",
            "Description: The dataset contains information on various states and local government areas in Nigeria, including total population, number of internally displaced persons (IDPs) in camps and host communities, number of returnees, and size of host communities. Each row represents a different local government area within the state of Adamawa, with data such as population figures, IDP numbers, and returnee statistics. The dataset provides a detailed breakdown of the humanitarian profile of these areas, highlighting the impact of displacement and the needs of the affected populations.\n",
            "Description: The dataset from the file \"afg_hno_pin_20243.xlsx\" contains information on people in need in Afghanistan for the year 2024, categorized by various population groups and locations. It includes data on education in emergencies, emergency shelter and non-food items, food security and agriculture, and health services. The data is presented in millions and percentages, detailing figures for different demographic groups such as women, children, adults, elderly, and people with disabilities. The dataset provides a comprehensive overview of the humanitarian needs in Afghanistan for the specified year.\n",
            "Description: The dataset contains information on humanitarian needs in various states and counties in South Sudan for the year 2024. It includes data on the number of internally displaced persons (IDPs), residents, returnees, and the total population in need. Additionally, it provides a breakdown by age and gender categories such as children, adults, elderly, girls, boys, women, and men. The data shows variations in needs across different regions, with some areas having higher numbers of IDPs and residents in need compared to others.\n",
            "Description: The dataset contains information on various regions in Mali, including demographic data such as population numbers categorized by age and gender, as well as targeted assistance figures. The data includes columns for region names, administrative codes, population numbers for different age and gender groups, and targeted assistance numbers. The dataset covers regions such as Bamako, Bandiagara, Bougouni, Douentza, Gao, and more, providing insights into the population distribution and assistance needs in these areas.\n",
            "Description: The dataset contains information on different administrative regions in Sudan, including Admin1 and Admin2 names and codes, as well as the number of people in need and targeted for assistance in each region. The data includes regions like Khartoum and North Darfur, with corresponding population figures for people in need and targeted for aid. The dataset provides a snapshot of the humanitarian needs in these regions, highlighting the scale of assistance required to support the affected populations.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/UNHCR_UKR_dataset3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Description: The dataset contains information on different provinces and territories, including various demographic and socio-economic indicators such as population, health, education, nutrition, and protection services. Each row represents a specific location with corresponding data values for different categories. The dataset includes columns for province and territory codes, zone of health, population breakdown by gender and age groups, as well as data on specific services provided in each location. The dataset seems to focus on areas within Kinshasa, providing detailed information on different zones and their corresponding services and population demographics.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset global_pcodes3.csv contains information on locations, administrative levels, P-Codes, names, parent P-Codes, and valid from dates. The data includes details such as country codes, admin levels, specific P-Codes for different regions within Afghanistan, corresponding names of these regions, parent P-Codes, and the date from which the data is valid. The dataset seems to focus on administrative divisions within Afghanistan, providing a structured format for organizing and referencing geographical information.\n",
            "Description: The dataset contains information on different clusters, organizations, organization types, regions, and districts. The data includes columns such as Cluster, Organization, Org Type, Region, and District. The dataset appears to list various organizations operating within different regions and districts, with each entry specifying the cluster to which the organization belongs, the organization's name, type, and the specific region and district where it operates.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (5,54) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various measures and indicators related to COVID-19 response for different countries and regions. It includes data such as government responses, containment measures, economic support, vaccination policies, confirmed cases, confirmed deaths, majority vaccinated status, population vaccinated, stringency index, and other related indices. The data is structured with columns representing different aspects of the response measures and indicators, with rows corresponding to specific dates and locations.\n",
            "Description: The dataset contains information on health institutions in Haiti, including details such as administrative divisions (adm1_fr, adm1_ht, adm2code, adm2_en, adm2_fr, adm3code, adm3_en, adm3_fr), institution name, category, institution type, institution code, latitude (LatDD), and longitude (LongDD). The data includes various health facilities like hospitals, clinics, and dispensaries, with information on their ownership status (public or private) and profit status (for-profit or non-profit). The dataset provides a comprehensive overview of health institutions in Haiti, their locations, and categorization.\n",
            "Processed 13.0% resources ...\n",
            "Description: The dataset contains information from the Displacement Tracking Matrix (DTM) Somalia Baseline Assessment Round 23, focusing on the mobility tracking exercise in Somalia's Jubaland, Hirshabelle, and South West States. The data, collected in February and March 2023, aims to quantify population categories, reasons for displacement, length of displacement, and needs within defined locations. The methodology involves two steps: Baseline 1 at the district level and Baseline 2 at the settlement level, with data collected through key informants' interviews and direct observations. The findings should be considered as estimates due to the evolving context of Somalia, including frequent shocks like flooding, drought, and conflict, which impact population dynamics. Limitations include the reliance on key informants' perceptions and the potential for missing or inaccessible locations.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Ethiopia Site Assessment Round 343.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 24160 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Ethiopia Village Assessment Sep23 R173.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 29616 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on movements and population figures in different regions, with columns representing data categories such as sources, official data, operational data, region, MAH/GC, dates, DTM, monitoring protection, and RRM. The data includes population numbers for regions like Diffa, Maradi, Niamey, Tahoua, and Tillaberi, along with corresponding dates. The dataset seems to be structured with missing values and headers that need further organization for clearer analysis.\n",
            "Description: The dataset contains information on various locations in Central African Republic, including the number of affected individuals, summary of needs, assistance provided, type of response, actors involved, gaps in response, evaluation status, and reasons for not conducting evaluations. The data includes details such as the prefecture, sub-prefecture, locality, number of affected individuals, types of assistance provided, and reasons for gaps in response. The dataset also indicates whether evaluations were conducted and reasons for not conducting evaluations in certain cases.\n",
            "Description: The dataset contains information on different regions in the Central African Republic, including the prefecture, sub-prefecture, year, cluster, population, and various indicators related to humanitarian needs such as PiN (People in Need), PiN Aigu, PiN IDP Site, PiN IDP FA, PiN Returnees, and PiN Hosts. The data also includes categories for levels of need ranging from None/Minimal to Catastrophic. The dataset covers multiple regions and provides detailed information on the population and humanitarian needs in each area for the year 2024.\n",
            "Description: The dataset contains information on organizations operating in different sectors within various provinces in a specific region. It includes columns for province code, province name, cluster, organization acronym, organization name, and organization type. The data shows multiple entries for different organizations operating in sectors such as nutrition, health, education, and water, sanitation, and hygiene (WASH). The organizations vary between international NGOs like Action Contre la Faim (ACF) and national NGOs like Association pour le Développement Economique et Social du Lac (ADESOL) and AIDER.\n",
            "Description: The dataset contains information on flood-affected households and individuals in different regions of Burkina Faso. It includes columns for region name, region code, number of affected households, and number of affected individuals. The data shows varying levels of impact across regions, with the Sahel region being the most severely affected with 1345 households and 6725 individuals impacted. The dataset provides a snapshot of the flood impact in different regions of Burkina Faso.\n",
            "Description: The dataset contains information on different administrative regions in Burkina Faso, including details such as administrative codes, population statistics, and various indicators related to protection, health, food security, education, and other sectors. The data is structured in rows and columns, with each row representing a specific administrative area and its corresponding data values. The dataset provides a comprehensive overview of the humanitarian needs and services in different regions of Burkina Faso.\n",
            "Description: The dataset contains information on health care-related incidents in Iran, including details such as the date, country, reported perpetrator, weapons used, number of attacks on health facilities, destruction and damage to health facilities, occupation of health facilities, health transportation status, looting/theft incidents, obstruction to health care, and harm to health workers (killed, injured, kidnapped, arrested, threatened, assaulted, sexually assaulted). The data also includes information on different categories of health workers affected, such as international health workers, national employees of aid agencies, and local health workers. Additionally, the dataset includes details on conflict-related violence, politically-related violence, COVID-19-related violence, Ebola-related violence, vaccination-related violence, and unique event IDs associated with each entry.\n",
            "Processed 14.000000000000002% resources ...\n",
            "Description: The dataset contains information on humanitarian response plans for various countries in 2023, including the number of people in need, people targeted, requirements in US dollars, funding received, and coverage percentage. The data includes countries like Afghanistan, Burkina Faso, Burundi, Cameroon, Central African Republic, Chad, Colombia, Democratic Republic of the Congo, and El Salvador. The coverage percentage ranges from 0.24 to 0.47, indicating the proportion of funding received compared to the requirements.\n",
            "Description: The dataset contains information on various organizations involved in humanitarian activities in Mali, including details such as organization acronym, name, type, program, funding source, and geographical location down to the commune and village levels. It also includes data on specific projects, start and end dates, activities, cash transfer amounts, and the number of individuals targeted, categorized by gender, age groups, and disabilities. The dataset appears to capture a wide range of humanitarian interventions and their beneficiaries, with a focus on food security programs.\n",
            "Description: The dataset contains information about schools in Sudan, including details such as school ID, name in Arabic and English, location, type, status, number of teachers, and student demographics. It also includes information on facilities like classrooms, electricity source, water supply, and sanitation facilities. The dataset covers both boys' and girls' schools in urban and rural areas, providing insights into the educational infrastructure in West Kordofan state.\n",
            "Description: The dataset contains information on the number of schools in Ethiopia categorized by educational level, including KG, pre-primary, primary, and secondary. The data is presented in a tabular format with columns for school types and corresponding counts. The dataset includes a total of 45,534 schools, with the majority being primary schools (36,323). The dataset also includes rows for subtotals and grand totals, as well as some empty rows and columns.\n",
            "Description: The dataset contains information related to the humanitarian situation in Ukraine, including figures such as total population, people affected, internally displaced persons (IDPs), people in need, and people targeted for assistance. It also includes data on funding requirements and coverage for various humanitarian response plans, as well as contributions to different funds. The dataset spans multiple dates in March 2022 and includes URLs to relevant reports and appeals.\n",
            "Description: The dataset contains information on various organizations involved in humanitarian activities in Nigeria, including their acronyms, types of organizations, operation types, project sectors, activities, status, states, LGAs, and response types. The data includes details such as the organization \"Action Against Hunger\" conducting activities related to WASH (Water, Sanitation, and Hygiene) in different locations like Borno, Yobe, and Maiduguri during July 2023. The dataset seems to focus on reporting ongoing humanitarian efforts, particularly in the WASH sector, with a nexus approach.\n",
            "Description: The dataset contains information related to humanitarian coordination activities in Afghanistan for July, August, and September 2023. It includes details on Operational Presence (OP) and Operational Capacity (OC) of organizations working in different districts. The dataset categorizes organizations based on their operational presence and capacity, aiming to identify gaps, avoid duplication of efforts, and plan for future humanitarian responses. Additionally, the dataset includes sheets listing actor acronyms by district, the number of humanitarian actors by cluster and district, and total unique actors with operational presence and capacity across all clusters. Protection actors' names have been intentionally removed for confidentiality reasons, and the dataset is described as a work in progress.\n",
            "Description: The dataset contains information about countries and territories, including various IDs, numerical codes, API IDs, and administrative details such as independence status and region. It also includes country names in multiple languages, latitude and longitude coordinates, income levels, currencies, and reference periods. The dataset covers a range of countries and territories, providing a comprehensive overview of their key attributes and characteristics.\n",
            "Description: The dataset contains information on emergency events such as floods, landslides, typhoons, and epidemics in various countries in the year 2024. Each entry includes a GLIDE number, emergency name, status (alert or ongoing), year, country name, country code (ISO3 and ISO2), and crisis type. The dataset provides details on events like floods in Sudan, Burkina Faso, and Guatemala, landslides in Ethiopia, a typhoon in China, and a cholera outbreak in Myanmar.\n",
            "Description: The dataset contains information related to humanitarian coordination activities in Syria between January and March 2023. It includes details on operational presence and capacity of organizations working in different districts, as well as a list of actors and clusters involved in humanitarian efforts. The dataset emphasizes the importance of knowing where organizations are working, what they are doing, and their capabilities to identify gaps, avoid duplication, and plan for future humanitarian responses. The information provided is considered a work in progress, with some details intentionally removed for confidentiality reasons.\n",
            "Processed 16.0% resources ...\n",
            "Description: The dataset contains information on PIN sectorial and intersectorial by municipality for the year 2023 in Colombia. It includes data such as municipality codes, department names, population figures, severity levels, and PIN values for various sectors like shelter, education, protection, violence against gender-based violence (VBG), child protection (CP), mines, road traffic (RT), health, water, sanitation, and food security. The data is structured in a tabular format with columns representing different attributes and values for each municipality in Antioquia, including metrics like severity levels and corresponding PIN values for different sectors.\n",
            "Description: The dataset contains financial data for various sectors such as Coordination, Early Recovery, Education, Food security, Health, Nutrition, Protection, and General Protection for the years 2021, 2022, and 2023. It includes budget figures, variations between years, and percentage comparisons. The data is presented in a tabular format with columns representing different aspects of the financial information for each sector.\n",
            "Description: The dataset contains information on internally displaced persons (IDPs) in various regions and provinces in Burkina Faso as of March 31, 20233. The data includes details such as the number of men, women, and children under 5 and over 5 years old, as well as total individuals in need. Each entry specifies the destination country, region, province, commune, and village or camp. Additionally, administrative codes and the date of data collection are provided for each entry.\n",
            "Description: The dataset contains information on flow monitoring in South Sudan from November 2022 to October 233. It includes details such as interview month, flow direction, registered refugees, transportation mode, departure and destination countries, reasons for travel, nationality, age and gender breakdown of individuals, total number of individuals, and indicators for migration, forced displacement, and return/relocation. The data captures both incoming and outgoing movements, with various travel reasons cited including family-related travel, seasonal movements, and celebrations. The dataset provides insights into migration patterns and reasons for movement within the specified time frame.\n",
            "Description: The dataset contains information on various organizations involved in humanitarian efforts in the Adamaoua region of Cameroon. It includes details such as organization acronyms, names, types, and the specific administrative areas they operate in. The dataset also lists the clusters and sectors each organization is involved in, such as shelter, education, health, logistics, and refugee response. The data provides insights into the diverse range of organizations working in the region and their respective focus areas within the humanitarian sector.\n",
            "Description: The dataset contains nutrition data for different provinces, territories, and health zones in the Democratic Republic of the Congo. It includes information such as prevalence rates of moderate acute malnutrition (MAM), severe acute malnutrition (MAS), and moderate acute growth retardation (MAG), as well as severity levels for these indicators. The dataset also includes data on the IPC-NUT severity, current IPC status, projected IPC status, and impact of malnutrition. The data is structured with columns for province codes, names, territory codes, territory names, health zone codes, and various nutrition indicators.\n",
            "Description: The dataset provided is an Excel file containing information related to organizations, activities, and locations in Burkina Faso for the months of July and August 20233. The data includes instructions on how to fill out the template, contact information for inquiries, and notes on the template's origin. The dataset also includes fields for organization names, implementing partners, funding sources, reporting months, activity statuses, and regions where activities are being implemented. The data seems to be structured for reporting and tracking various humanitarian activities in Burkina Faso.\n",
            "Description: The dataset contains information on different provinces in the Democratic Republic of the Congo, including their administrative codes, methodology used for monitoring, number of internally displaced persons (IDPs) assessed, and number of returnee individuals. The data shows varying numbers of IDPs and returnees across different provinces, with some provinces having significantly higher numbers compared to others. The methodology used for monitoring is primarily \"Mobility Tracking Light,\" with some variations in the number of individuals assessed and returned.\n",
            "Description: The dataset contains information on regions in Somalia affected by drought, including details such as administrative codes, districts, operational priorities, total affected population, targeted population, and reached population. The data includes regions like Middle Shabelle, Lower Shabelle, Lower Juba, Gedo, Awdal, and Banadir, with corresponding figures for total affected, targeted, and reached populations in each region. The dataset provides insights into the impact of drought in different areas of Somalia and the progress in reaching targeted populations with relief efforts.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC – Mobility Tracking Atlas – Round 3 – October 20233.xlsx appears to be empty, as indicated by the absence of any columns or index values in the provided sample. Further analysis or investigation is needed to determine the content and structure of the data within the file.\n",
            "Processed 18.0% resources ...\n",
            "Description: The dataset titled \"DTM Libya Migrants Baseline Assessment Round 483\" contains information on migrants in Libya, including the total number of migrants (704,369), the number of Baladiyas (municipalities) and communities hosting migrants, and data on migrants by nationality and region. The data includes details such as the period of assessment (May-June 2023) and specific figures for migrants in different regions of Libya, such as Al Jabal Al Akhdar in the East region.\n",
            "Description: The dataset contains information on repatriation statistics for September 2017 to October 20233, with columns including Province, Commune, Colline, Sous Colline, Sex, Convoy date, Male, Female, Country of residence, and Overall Total. The data shows the number of males and females repatriated from different locations to Tanzania during the specified period. The dataset seems to have some missing values and duplicates in the Male and Female columns.\n",
            "Description: The dataset contains information on community needs identification for flood response in Pakistan, specifically focusing on various provinces and districts. It includes data on the total population, households, internally displaced persons (TDPs), age and gender demographics, living conditions, and returnees. Each entry provides details such as reported date, province, district, population breakdown by age and gender, TDP households in different living arrangements, and total population including both host communities and external TDPs. The dataset offers a comprehensive overview of the impact of floods on different communities in Pakistan, highlighting the need for targeted response and support efforts.\n",
            "Description: The dataset contains information on asylum applications by asylum seekers originating from Cook Islands. It includes data such as the year, country of origin code, country of asylum code, country names, procedure type, procedure name, application type code, application type, application data type, application data, application average persons per case, and the number of applications. The data shows that in 2010, there were 5 asylum applications from Cook Islands to South Africa under the government procedure for new applications, with an average of 0 persons per case.\n",
            "Description: The dataset contains information on provinces, territories, and various indicators such as SECAL, AME, ABRI, SANTE, EDUCATION, NUTRITION, WASH, CCCM, Protection générale, Protection de l'enfant, VBG, and Total Budget across different regions like Ituri and Nord-Kivu. It includes data on population figures, budget allocations, and specific needs in different sectors. The dataset also includes codes for provinces and territories, as well as global targets for certain indicators.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM DRC - BA North Kivu October 2023 - R103.xlsx appears to be empty, with no columns or index values present in the DataFrame.\n",
            "Description: The dataset contains information on immunization campaigns impacted by various factors such as COVID-19 across different countries. It includes details such as the country code, country name, campaign vaccine, planned start and end dates, status of the campaign, target age group, estimated target population, campaign type, and geographic scale. The data shows a variety of campaigns, including those related to poliovirus, measles, rubella, and meningitis, with details on whether they were postponed, reinstated, or on track. The dataset covers both national and subnational campaigns, providing insights into the impact of external factors on immunization efforts.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM DRC BA South Kivu Oct 2023 R113.xlsx appears to be empty, as the DataFrame contains no columns or index values.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM DRC - BA Tanganyika October 2023 - R163.xlsx appears to be empty, as it contains no columns or index values.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/NER_Carte_sanitaire_du_Niger3.xlsx appears to be empty, with columns including Orgunit levels 1 to 6, Dhis2 ID, Period, and Population Total (P1). There are no rows of data present in the dataset, indicating that it does not contain any information at the moment.\n",
            "Processed 20.0% resources ...\n",
            "Description: The dataset contains information about educational establishments in Niger, including details such as administrative codes, regions, departments, communes, establishment names, inspection status, types of establishments, socio-economic status, educational systems, academic years, and student and teacher demographics. The data includes the number of male and female students in different grade levels, total student enrollment, and total teaching staff holding classes. The establishments are predominantly public, traditional, and located in rural areas, focusing on basic education.\n",
            "Description: The dataset contains information on estimated maximum numbers of internally displaced persons (IDPs) and returnees in various oblasts of Ukraine as of September 30, 2023. The data includes columns for the round number, country name, country code, reported date, oblast name, oblast pcode, estimated max. de facto IDPs present, and estimated max. returnee population present. The dataset highlights that estimates in certain oblasts may be underrepresented due to limited coverage of government-controlled areas and a limited number of respondents reached. Additionally, it mentions that respondents currently on short-term trips outside their places of current residence are considered.\n",
            "Description: The dataset contains information on sectors, organizations, project interventions, and administrative details related to humanitarian and development activities in Haiti. It includes columns for sectors in French and English, organization acronyms and names in French and English, organization types, domains, administrative codes and names, as well as project or intervention descriptions. The data showcases various sectors such as protection, health, food security, and education, along with details of organizations like FONDATION TERRE BLEUE and SOLIDARITÉ POUR UNE JEUNESSE RÉSILIENTE. The dataset appears to focus on activities in Nippes and Artibonite regions, with a particular emphasis on humanitarian and development initiatives following the earthquake of August 14, 2021.\n",
            "Description: The dataset contains demographic data for different regions in Burundi, including information on population totals, gender breakdowns, age groups, and administrative divisions. Each row represents a specific region, with columns indicating various demographic variables such as total population, gender-specific populations, age group distributions, and administrative codes. The dataset includes regions from Bubanza and Bujumbura Mairie, with details on population counts for different age groups and genders. The data provides a comprehensive overview of the demographic composition of these regions in Burundi.\n",
            "Description: The dataset contains information related to various administrative regions in the DRC, specifically focusing on the Ituri province. It includes data on different territories, health zones, population statistics, key informants, and various metrics related to displaced persons and returnees. The dataset provides details on villages assessed, known villages, coverage of displaced persons in family arrangements, changes in displacement over time, and coverage by the CCCM mechanism. The data covers a range of time periods, including arrivals and returns of displaced persons over 18, 24, and 36 months. Additionally, it includes information on different health zones within the region and their respective populations.\n",
            "Description: The dataset contains information on returnee populations in different regions of Ukraine as of September 30, 2023. It includes data such as the round number, country name, country code, reported date, region name, region code, and estimated maximum returnee population present in each region. The dataset shows varying numbers of returnees across different regions, with the highest estimated population in Kyiv at 1,042,000 and the lowest in Chernivetska at 19,000.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Burundi Baseline Assessment Round 733.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 16586 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/suivi-du-bilan_2022_nutrition3.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/suivi-du-bilan_2022_nutrition3.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/suivi-du-bilan_2022_nutrition3.xls\n",
            "Description: The dataset contains information related to IDPs (Internally Displaced Persons) and returnees in South Sudan, specifically in the Central Equatoria region. It includes data on various locations, IDP households, IDP individuals, arrival periods, reasons for displacement, housing conditions, returnee households, returnee individuals, and reasons for return. The data also covers details on IDPs and returnees who have previously been abroad, as well as information on displacement within South Sudan and outside the country. Additionally, the dataset includes details on relocated populations, houses occupied by non-owners, and consistency of data observations. The dataset provides a comprehensive overview of the displacement and returnee situation in the specified region.\n",
            "Description: The dataset contains information on the number of internally displaced persons (IDPs) and returnees in different regions of Libya. The data includes the regions of Benghazi, Misrata, Tripoli, Sirt, Sebha, Murzuq, Almargeb, Ubari, and Ejdabia. For each region, the dataset provides the number of IDPs in households (HH) and individuals (IND), as well as the percentage of IDPs. Similarly, it includes the number of returnees in households and individuals, along with the percentage of returnees. The dataset shows variations in the number of IDPs and returnees across different regions in Libya.\n",
            "Description: The dataset contains information on various administrative regions in Nigeria, including columns for Admin1 Name, Admin1 Pcode, Admin2 Name, Admin2 Pcode, Host Community, IDPs in Camp, IDPs in Host Community, Returnees, and Total People in Need. Each row represents a different region within Adamawa, with data on the number of internally displaced persons (IDPs) in camps and host communities, returnees, and the total population in need. The dataset provides a detailed breakdown of these figures for different regions within Adamawa, showcasing the distribution of displaced populations and those requiring assistance in the area.\n",
            "Processed 22.0% resources ...\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Mozambique Site Assessment Round 223.xls\n",
            "Description: The dataset contains information from a baseline assessment in Madagascar, with columns representing various demographic and migration-related data such as household and individual counts, migration patterns from 2009 to 2018, reasons for displacement (e.g., economic, seasonal, other), means of transportation, and presence of return migration. Each row corresponds to a specific location within the Androy region, detailing characteristics like the number of households, individuals, and migrants, as well as their movements and reasons for displacement. The dataset also includes information on return migration, including the number of returning households and individuals, as well as their destinations and reasons for return.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM DRC BA Kasai Central Dec2020 R73.xlsx appears to be empty, with no columns or index values present.\n",
            "Description: The dataset contains information on migrant presence monitoring in Lebanon, specifically focusing on the number of migrants by nationality in different governorates and districts. The data includes nationalities such as Ethiopian and Bangladeshi, with corresponding migrant population numbers. For instance, in Beirut district, there are 9515 Ethiopian migrants and 7423 Bangladeshi migrants. The dataset provides a breakdown of migrant numbers across various regions in Lebanon, offering insights into the distribution of migrant populations within the country.\n",
            "Description: The dataset provided is in an Excel file format and contains information related to organizations, projects, activities, and humanitarian response plans for the year 2023. The data includes columns such as organization acronym, organization name, project type, project sector/domain, project status, start and end dates, number of direct beneficiaries, project location details, presence in the community, accountability mechanisms, information provision, community participation, feedback collection, AAP institutionalization, and evaluation mechanisms used. The dataset also includes details of specific projects undertaken by an organization, such as Action Contre La Faim Espagne (ACF-E), including their activities, beneficiaries, project timelines, and completion status.\n",
            "Description: The dataset contains information on health facilities, including the region, district, health facility name, type, latitude, and longitude. It includes data for various regions such as Banadir, Nugaal, and Bakool, with details on different health centers within each region. Some entries have missing latitude and longitude values. The dataset provides a snapshot of the distribution and types of health facilities across different regions.\n",
            "Description: The dataset contains information on key figures related to the Somalia drought, including the country, date, total affected population, total targeted population, percentage targeted, total reached population, percentage reached, food insecurity, malnutrition (SAM, MAM, GAM), internal displacement, school dropout, livestock deaths, water insecurity, and disease outbreaks (AWD/Cholera, Measles). The data shows changes over time in these metrics, with varying levels of impact and response efforts. The dataset also includes source links for additional information.\n",
            "Description: The dataset contains information on various prefectures and sub-prefectures, along with the number of households and individuals classified as internally displaced persons (IDPs) and returnees, both internally and from other countries. The data includes columns for prefecture names and codes, sub-prefecture names and codes, as well as counts of IDP households and individuals, and returnee households and individuals. The dataset provides specific figures for each category across different regions, such as Haute-Kotto and Ombella M'Poko.\n",
            "Description: The dataset contains information on voyages undertaken by ships under the Black Sea Grain Initiative from August 1, 2022, to July 17, 2023. The data is sourced from the United Nations Joint Coordination Centre website and is presented in two tabs: DATA and DATA_HXL. The columns in the dataset include information such as status, outbound sequence, and departure port. The dataset provides details on shipments, voyage sequences, and departure ports related to the Black Sea Grain Initiative vessel movements.\n",
            "Description: The dataset titled \"DTM Sudan Migrants Master List Round 63\" contains information on foreign nationals in Sudan collected during Round 6 of the Displacement Tracking Matrix (DTM) Dataset. The data includes details on 558,993 migrants from 120,941 households, spread across 17 states, 122 localities, and 708 locations in Sudan. The dataset also mentions the data collection period from January to March 2023 and indicates that it was published for public access.\n",
            "Description: The dataset contains information on returnees in Sudan, including details such as interview dates, location, household and individual counts, return timelines, shelter types, and credibility ratings. The data includes various locations in Darfur and other regions, with distinctions between urban and rural areas. Returnee numbers vary across different locations, with details on the types of shelters they are residing in. The dataset also includes a credibility rating for each entry, indicating the reliability of the information provided.\n",
            "Processed 23.0% resources ...\n",
            "Description: The dataset titled \"MOBILITY TRACKING Round 5\" contains information from the Displacement Tracking Matrix (DTM) Dataset Round 5 for Sudan. The data includes details on 3,779,487 internally displaced persons (748,100 households) identified across 17 states, 127 localities, and 882 locations in Sudan. The data was collected between January and March 2022 and was published in December 2022 by the International Organization for Migration (IOM).\n",
            "Description: The dataset contains information on internal displacements associated with disasters, with columns including country ISO code, country name, year, start and end dates of the event, event name, hazard category and sub-category, type, subtype, new and total displacement numbers, and glide numbers. The data includes details on various weather-related events such as flooding, landslides, hailstorms, and wildfires in Zimbabwe from 2013 to 2014. The dataset provides insights into the number of new and total displacements caused by different types of hazards during this period.\n",
            "Description: The dataset titled \"Internal Displacements (New Displacements) – IDPs3.csv\" contains information on new and total displacement figures for different countries over the years. The data includes columns such as iso3 (country code), country_name, year, new_displacement, new_displacement_rounded, total_displacement, and total_displacement_rounded. The data shows that for Turkmenistan from 2010 to 2014, there were no new displacements recorded, with a total displacement figure of 4000 for each year.\n",
            "Description: The dataset contains information on various districts in Mongolia, including demographic data such as population breakdown by age group, household information, vulnerability indicators, reasons for arrivals and departures, employment types, health reasons, mobility challenges, and actions preferred to address these challenges. The data also includes details on individuals' registration status, reasons for not registering, and additional comments. Each entry includes district-specific data, such as population demographics, reasons for mobility, employment sectors, and challenges faced.\n",
            "Description: The dataset contains information on various administrative regions in Ethiopia, including details such as the names and codes of different administrative levels, total population figures, and specific targets related to different categories like IDPs (Internally Displaced Persons) in sites, IDPs out of sites, returning IDPs, affected non-displaced individuals, and food targets. Each row represents a different administrative region within the North Western region of Tigray, Ethiopia, with corresponding population and target data. The dataset provides a comprehensive overview of the population and targeted assistance in these regions for various sectors.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Burundi Multi Sectoral Location Assessment Round 33.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 25895 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on various countries related to COVID-19 data, restrictions, and other relevant factors. It includes details such as population, cumulative cases, deaths, economic exposure, malnutrition estimates, travel restrictions, funding, testing rates, food insecurity, vaccination status, refugees, severity index, and more. Each entry provides specific data for a country, along with the latest news on international and internal restrictions, travel guidelines, and other important updates. The dataset covers a range of countries including Aruba, Afghanistan, Angola, Argentina, Burundi, Burkina Faso, Bangladesh, Bolivia, and Brazil, offering insights into the COVID-19 situation and related measures in each country.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Ethiopia - VAS - Tigray - R163.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 30801 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on various districts in Somalia, including their names, Pcodes, targeted individuals, and access severity levels. The access severity levels range from \"Hard to Reach\" to \"low access constraints\" and \"moderate access constraints.\" The data provides insights into the challenges faced in accessing services in different districts of Somalia based on the severity levels indicated.\n",
            "Description: The dataset contains information on flood-related incidents in Mali, with columns representing different categories such as region, circle, number of affected people, deaths, injuries, destroyed houses, livestock, damaged water points, destroyed latrines, and affected households. The data includes details such as the number of affected individuals, houses destroyed, livestock, and households impacted in various regions and circles of Mali. The dataset provides a snapshot of the impact of flooding in different areas, highlighting the extent of damage and the number of people affected by these incidents.\n",
            "Processed 25.0% resources ...\n",
            "Description: The dataset contains information on the operational priority, overall affected population, target population, and reached population in various counties in Kenya affected by drought. The data includes columns for location, administrative code, operational priority, total affected population, total target population, and total reached population. The dataset shows that different counties have varying levels of affected, targeted, and reached populations, with operational priorities ranging from 1 to 2.\n",
            "Description: The dataset \"Food basket change3.csv\" contains information on the change in food basket cost for various countries. It includes columns for the ISO3 country code, country name, and the percentage change in the food basket cost. Some entries have missing values for the change in food basket cost. The dataset provides insights into how the cost of food baskets has changed in different countries over a certain period.\n",
            "Description: The dataset contains information on humanitarian needs in Ethiopia for 2021, with data organized by administrative regions and sub-regions. It includes details such as the total number of people in need, broken down by categories such as children (female and male), adults (female and male), older persons (female and male), and people with disabilities. The dataset provides a comprehensive overview of the humanitarian situation in different areas of Ethiopia, highlighting the specific needs of vulnerable populations in each region.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Ethiopia - SA - Tigray - R333.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 21672 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on verified security incidents, including incident ID, date, location details such as country, region, district, and city, as well as the number of nationals and internationals killed, wounded, and kidnapped. It also includes data on the total number of individuals affected, gender distribution, means of attack, attack context, location coordinates, motive, actor type, and actor name. The dataset appears to have a mix of verified and archived incidents, with details on the source of verification provided.\n",
            "Description: The dataset contains information from the Kazakhstan International Migrant Workers Survey - Round 23. It includes data on various demographic and migration-related variables such as age, marital status, education level, employment status, migration intentions, challenges faced while working, reasons for leaving previous jobs, current income, satisfaction levels, financial behaviors, and barriers to accessing health and education services. The dataset also captures details on the respondents' interactions with their national and community relationships, as well as their reasons for considering or not considering returning to their home country.\n",
            "Description: The dataset contains information from a survey conducted in Nigeria, specifically focusing on returnees in different locations within the country. The data includes details such as the date of the survey, population type, geographical locations (state, LGA, ward), reasons for displacement and return, settlement types, arrival numbers in different years, demographics (infants, children, youth, adults, elderly), and total male and female counts. The dataset provides insights into the demographics and characteristics of returnee populations in various areas of Nigeria, highlighting factors such as communal clashes and improved security influencing their return to host communities.\n",
            "Description: The dataset contains information on locations in Ethiopia affected by drought, including administrative codes, names, regions, operational priorities, total affected individuals, targeted individuals, and reached individuals. The data includes details such as the location's priority level, total affected population, targeted population, and the number of individuals reached in various regions and zones within Ethiopia.\n",
            "Description: The dataset contains information on estimated numbers of internally displaced persons (IDPs) in various counties in Kenya due to drought-related issues. The data includes columns for County name, County p-code, IDP estimates, and comments on the data source. The estimates range from 29,112 IDPs in Isiolo to 205,014 IDPs in Garissa. The data appears to be sourced from reports on human mobility and displacement tracking by organizations such as the Displacement Tracking Matrix.\n",
            "Description: The dataset contains information on IDPs dispersed in host communities in Nigeria, specifically in regions like Benue and Kaduna. It includes details such as survey round, survey date, population types, location types, site details, displacement reasons, living conditions, access to services like healthcare and education, livelihood challenges, and coping mechanisms. The data also covers aspects like shelter conditions, water and sanitation facilities, food access, health problems, and access to healthcare services. Additionally, it provides insights into education services available, school attendance, obstacles to education, and livelihood support. The dataset captures the vulnerabilities, needs, and experiences of IDPs in these regions.\n",
            "Processed 27.0% resources ...\n",
            "Description: The dataset contains information on site assessments in Nigeria, specifically focusing on internally displaced persons (IDPs) dispersed in host communities in various states and local government areas. The data includes details such as reported dates, survey dates, population types, site names, households, individuals, reasons for displacement, settlement types, total number of IDPs, arrival statistics for different years, and demographic breakdowns by age and gender. The dataset provides a comprehensive overview of IDP situations in Nigeria, highlighting factors such as communal clashes and insurgency as reasons for displacement.\n",
            "Description: The dataset contains information on various districts in Yemen, including the governorate, district names and codes, number of internally displaced persons (IDPs) and estimated resident populations. It also includes data on persons with disabilities, population breakdown by age and gender, and totals for men/boys and women/girls. The dataset covers multiple districts within the Ibb governorate, providing detailed demographic information for each district.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (3,6,7,10,11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains transaction data from the file \"transactions3.csv\" with columns including Month, Reporting org id, Reporting org name, Reporting org type, Sector, Recipient country, Humanitarian indicator, Strict indicator, Transaction type, Activity id, Net money, and Total money. The data includes information on transactions made by the AECID Spanish Agency for International Development Cooperation in various sectors and countries during January 2020. The transactions involve commitments with different monetary values allocated to specific activities.\n",
            "Description: The dataset from the file \"GHO-mid-year-update-20233.xlsx\" contains information related to Humanitarian Action 2023, including details such as the date of 27/06/2023 and the source being Humanitarian Action. The dataset appears to have columns labeled \"Unnamed: 0\" and \"Unnamed: 1\" with some rows containing NaN values.\n",
            "Description: The dataset contains information on drought-induced displacement, including regions, zones, woredas, and IDP estimates in Ethiopia. The data includes columns such as Region, Region p-code, Zone, Zone p-code, Woreda, Woreda p-code, and IDP estimates. The dataset provides details on the number of internally displaced persons (IDPs) in different regions, zones, and woredas due to drought.\n",
            "Description: The dataset from the file \"Haiti Floods - 20233.xlsx\" contains information on the impact of floods in different departments of Haiti. It includes data on the number of affected households, individuals evacuated or displaced, deaths, missing persons, injuries, houses flooded or damaged, and livestock losses. The data is organized by department, such as Ouest, Sud-Est, Nippes, Grand’Anse, Nord-Ouest, Centre, and Sud. The Ouest department has the highest number of affected households and individuals, while the Sud department reported one death and one missing person. The dataset provides a snapshot of the impact of the floods in various regions of Haiti.\n",
            "Description: The dataset contains information on site assessments in North Kivu, Democratic Republic of the Congo, conducted in July 2019. It includes details such as site names, assessment dates, site statuses, reasons for displacement, site types, GPS coordinates, administrative codes, number of households and individuals, place of origin of IDP groups, presence of management committees and agencies, reporting of security incidents, and access to various services like WASH, education, shelter, and health. The data also indicates the round number of the assessment.\n",
            "Description: The dataset contains information on different levels of access severity, ranging from Very Low Access Severity (Level 1) to Very High Access Severity (Level 5). Each level is accompanied by a description of the impact of humanitarian access constraints on partners' ability to reach People in Need (PiN) and on PiN's ability to access humanitarian assistance. The dataset also includes formulas for determining the access severity level based on a numerical value.\n",
            "Description: The dataset titled \"DTM Afghanistan Settlements Round 16 Baseline Assessment3.xlsx\" contains information from DTM Afghanistan's 16th round of the Baseline Mobility Assessment (BMA) conducted between September and December 2022. The data provides estimates of IDP, returnee, out-migrant, and cross-border returnee population figures at the community level. The dataset includes columns with unidentified data (Unnamed: 0 to Unnamed: 9) and a section with important information about the dataset.\n",
            "Description: The dataset contains information related to severity constraints on access in Haiti, specifically focusing on different administrative levels such as country, departments, communes, and specific zones. The data includes columns for ISO codes, administrative names and codes, specific zones, severity levels reported by different entities (UN, ONGI, ONGN), and an overall severity rating. The dataset seems to be structured with rows representing different locations in Haiti and their corresponding severity ratings for access constraints.\n",
            "Processed 28.999999999999996% resources ...\n",
            "Description: The dataset from the file \"3W_BDI_Mar20223.xlsx\" contains information on different provinces, sectors, organization acronyms, organization descriptions, and organization types. The data includes details such as the names of provinces, sectors like food security and child protection, and various organizations operating within these sectors. The organizations are categorized based on their type, such as private organizations, foundations, and national NGOs. The dataset provides a comprehensive overview of the organizations working in different sectors across various provinces in Burundi.\n",
            "Description: The dataset contains information on various organizations operating in the education sector in Afghanistan. It includes details such as the country code, sector, organization name, organization type, and the date. The organizations listed range from international non-governmental organizations (INGOs) to national non-governmental organizations (NNGOs). The data seems to be structured with columns for country code, sector, organization name, organization type, and date, with specific entries for each organization within the education sector in Afghanistan.\n",
            "Description: The dataset contains information on incidents related to health facilities and workers in various countries, including details on the date, country, reported perpetrator, weapons used, number of attacks on health facilities, and impact on health workers. The data includes categories such as destruction, damage, occupation, abduction, assault, and threats to health workers, as well as details on the type of violence involved (conflict-related, politically-related, COVID-19-related, etc.). Each entry provides specific information on the incident, such as the type of weapon used, the number of health workers affected, and the outcome of any kidnappings or arrests.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Burundi- Muyinga, Cankuzo, Makamba, Ruyigi, Rutana, Rumonge: Operational Presence3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/header_footer.py:48: UserWarning: Cannot parse header or footer so it will be ignored\n",
            "  warn(\"\"\"Cannot parse header or footer so it will be ignored\"\"\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on populated places in Iraq, including administrative divisions such as country, governorate, district, and sub-district levels. Each entry includes details such as place names in English and Arabic, location coordinates (longitude and latitude), and population estimates. The dataset provides a comprehensive overview of various populated places in Iraq, their administrative hierarchy, and population sizes.\n",
            "Description: The dataset contains information related to events in South Sudan, including details such as event dates, locations at various administrative levels, population demographics, displacement information, movement triggers, arrival and departure details, needs assessment, and other sector needs. The data includes assessments of affected populations, household and individual counts, age and gender breakdowns, shelter conditions, modes of transport, movement triggers, conflict sub-types, safety assessments, enroute issues faced, arrival and departure locations, future relocation plans, and reasons for people staying behind. The dataset also captures information on sector-specific needs such as food, shelter, non-food items, water, sanitation, health, and protection needs.\n",
            "Description: The dataset contains information on attacks on vaccination campaigns from 2016 to 2023, including details such as the date, country, reported perpetrator, weapons used, number of attacks on health facilities, health workers affected (killed, injured, kidnapped, arrested), and other related indicators. The data includes specific incidents in countries like Afghanistan, Pakistan, and Nigeria, with information on the perpetrators and the outcomes of the attacks. The dataset provides a comprehensive overview of the challenges and threats faced by health workers and vaccination campaigns during this period.\n",
            "Description: The dataset contains information on educational institutions in the Basse Kotto region of the Central African Republic. It includes details such as the name of the establishment, type of institution, geographical coordinates, functionality, number of classrooms, teachers, students, infrastructure details, and general condition of the buildings. The data also includes information on water sources, sanitation facilities, and any additional remarks. The dataset consists of multiple rows representing different educational institutions in the region, each with specific attributes and characteristics outlined in the columns.\n",
            "Description: The dataset contains information related to site assessments in Malawi following Cyclone Freddy, including details on survey dates, site locations, demographics, services provided, reasons for displacement, population breakdown, intended return areas, shelter and non-food items, water sanitation and hygiene, health services, education access, protection of vulnerable groups, communication, and camp management. Data includes reported dates, country details, survey rounds, sex of data collectors, site IDs, site names, region and district information, services provided at sites, population demographics, mobility, camp management, shelter types, NFI needs, water and sanitation availability, health services, education access, protection measures, communication topics, travel opportunities, population trends, and more.\n",
            "Description: The dataset contains information on IFRC appeals data for Uzbekistan, including details such as appeal ID, name, type, status, sector, funding amounts, start and end dates, and country information. The data includes appeals related to population movement, explosions, drought, and general assistance in Uzbekistan. The dataset also includes details on the organization involved, such as the Red Crescent Society of Uzbekistan, as well as information on the region, beneficiaries, and confirmation status.\n",
            "Description: The dataset contains information on various organizations and their activities related to food security in different governorates and communities in Gaza. The data includes details such as the cluster (sector name), organization names, partners involved, specific activities, governorates, and communities targeted. Organizations like Union of Agricultural Work Committees, Oxfam, and Save the Children are mentioned, along with their respective projects and locations. The activities mentioned focus on rehabilitation and provision of productive assets and inputs for farmers, herders, breeders, and fishers using the BBB approach, as well as providing direct access to food. The dataset provides a detailed overview of the efforts being made in the region to address food security issues.\n",
            "Processed 31.0% resources ...\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/displacement_data6.csv contains information on displacement events, including conflict and disaster-related internal displacements and stock displacements. The data includes columns such as ISO3 country code, country name, year, conflict stock displacement, conflict internal displacements, disaster internal displacements, and disaster stock displacement. The dataset provides details on the number of individuals affected by these displacement events, both in raw numbers and standardized formats. The example entry in the dataset pertains to Wallis and Futuna Islands in 2021, with some missing values for displacement data.\n",
            "Description: The dataset contains information on displacement data for Turkey from 2008 to 2016. It includes columns for ISO3 country code, country name, year, conflict stock displacement, conflict internal displacements, disaster internal displacements, and disaster stock displacement. The data shows the number of individuals affected by displacement due to conflict and disasters, with some entries having raw data values as well. The dataset provides a detailed breakdown of displacement figures for each year, highlighting the impact of conflicts and disasters on internal displacement in Turkey over the specified period.\n",
            "Description: The dataset contains information about humanitarian organizations' responses in Malawi, specifically detailing activities, locations, beneficiaries reached, and dates of distribution. Save the Children, an INGO, is shown to have distributed various items such as hygiene products, food, and supplies for child protection in different districts and camps in Malawi. The data includes details on the quantities and types of items distributed, as well as the specific locations and wards where the distributions took place. The dataset also includes information on the start dates of the distributions and any additional comments related to the activities.\n",
            "Description: The dataset contains information on Interagency Response Plans in 20233, with columns including the country name, ISO alpha-3 codes, plan type, In Need data, and whether the data is included in the Data Explorer PIN. The data includes countries such as Afghanistan, Burkina Faso, Burundi, Cameroon, Central African Republic, Chad, Colombia, Democratic Republic of Congo, and El Salvador, along with their respective population in need for humanitarian response plans.\n",
            "Description: The dataset contains information related to humanitarian coordination activities in Afghanistan for the months of October, November, and December 2022. It includes details on operational presence (OP) and operational capacity (OC) of organizations working in different districts, categorizing them based on their ability to deliver humanitarian services. The dataset also includes sheets listing actor acronyms by district, with protection actors' names removed for confidentiality. Additionally, there are sheets providing the unique number of humanitarian actors by cluster and district, as well as the total number of unique actors with operational presence (ALL_OP) and operational capacity (ALL_OC) across all clusters and districts. The dataset is described as a work in progress.\n",
            "Description: The dataset contains information on various districts, including their population, number of tweets, number of images, help request reports, impact score, and impact category. The districts have been categorized based on their help request impact score, with most falling under the \"Very High\" impact category. The data provides insights into the level of help requests and impact in each district, with some districts showing higher levels of help requests and impact compared to others.\n",
            "Description: The dataset contains information on COVID-19 vaccine distribution in various countries, including the WHO region, country name, country code, participant status, and the number of doses available for different vaccine types such as AstraZeneca/SII, AstraZeneca/SKBio, and Pfizer-BioNTech. The data includes details on the indicative distribution of doses for each vaccine type in different countries within the WHO regions.\n",
            "Description: The dataset contains information on various countries, including their population, COVAX forecast and delivery details for different COVID-19 vaccines. It includes columns such as Country, ISO3 code, SFP/AMC status, population, forecasted and delivered doses for different vaccines, and coverage percentages. Some countries have received vaccines from specific sources, with corresponding URLs provided. The dataset also includes deprecated columns related to other delivered doses and their sources.\n",
            "Description: The dataset contains information on death rates in various regions of Somalia from 2014 to 2018. It includes data such as the state, region, district, crude death rate, and under 5 years death rate. Each entry specifies the state, region, district, and corresponding death rates for both crude deaths and deaths under 5 years of age. The dataset provides a snapshot of mortality rates in different areas of Somalia over the specified time period.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Turkiye & Syria Earthquakes - OCHA SitReps3.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Turkiye & Syria Earthquakes - OCHA SitReps3.google sheet\n",
            "Description: The dataset contains information related to humanitarian activities in Iraq from January to December 2022. It includes metadata such as the month of reported activity in ISO 8610 format, location details specifying IDP in-camp, IDP out-of-camp, or informal settlement locations, administrative levels ranging from Admin1 to Location, and the Humanitarian Response Plan (HRP) associated with partner activities. The data is structured in a long table format and is stored in an Excel file located at the specified path.\n",
            "Processed 32.0% resources ...\n",
            "Description: The dataset contains information on different municipalities within the Amazonas department, including their DIVIPOLA codes, population in 2022, severity level based on the Joint Intersectoral Analysis Framework (JIAF) scale, and the number of people in need (PiN). The severity levels range from 2 to 4, with corresponding PiN values provided. The dataset includes municipalities such as El Encanto, Puerto Arica, Tarapacá, La Victoria, Miriti - Paraná, Puerto Santander, Puerto Nariño, La Pedrera, and La Chorrera.\n",
            "Description: The dataset contains information on conflict-induced displacements in Afghanistan for the year 2022. It includes details such as the newly displaced population due to conflict, compiled by OCHA sub offices based on inter-agency assessments. The data provides a snapshot as of January 5, 2023, with the note that the numbers are subject to change as new assessment figures become available. Additionally, the dataset includes fields such as the date of displacement, province code and name of origin and displacement, as well as district code and name of origin for internally displaced persons (IDPs).\n",
            "Description: The dataset contains information on COVID-19 cases in different governorates of Palestine, including the number of cases, deaths, recoveries, and active cases. The data includes details such as the governorate name, total cases, death cases, recovery cases, active cases, date of reporting, today's cases, vaccine ratio, and the proportion of fully vaccinated individuals. The dataset shows the latest statistics for each governorate, with Gaza strip having the highest number of cases. The dataset provides insights into the COVID-19 situation in Palestine, highlighting key metrics for monitoring and analysis.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/herams_nampula-db_qtr-2-2022_hxl3.xlsx ... Error code: 400 - {'error': {'message': \"Invalid 'messages[1].content': string too long. Expected a string with maximum length 1048576, but got a string with length 1183856 instead.\", 'type': 'invalid_request_error', 'param': 'messages[1].content', 'code': 'string_above_max_length'}}\n",
            "Description: \n",
            "Description: The dataset contains information on the number of children aged 6-59 months admitted for treatment of severe acute malnutrition (SAM) in Afghanistan. The data includes details such as the geographic area, time period, observed values, data sources, and observation status. The dataset seems to be related to humanitarian action for children, specifically focusing on health care for women and children. The data is structured with columns for various indicators, units of measure, and targets. The dataset appears to be sourced from UNICEF Situation Reports and includes information up to September 2020.\n",
            "Description: The dataset contains information on various departments in Colombia regarding indicators related to acute malnutrition in children under five years old. It includes data such as the number of low birth weight cases, cases of acute malnutrition, mortality associated with malnutrition, population statistics, and rates of malnutrition and low birth weight. The data is organized by department and includes values for each indicator for the specified reporting week.\n",
            "Description: The dataset contains information on various indicators related to food security and nutrition in different municipalities within the department of Antioquia, Colombia. The data includes population demographics, birth statistics, malnutrition indicators, and severity clusters. Each row represents a different municipality and includes details such as total population, population under 5 years old, number of live births, and specific indicators related to food security and malnutrition. The dataset also includes information on gender-specific populations and specific needs for different demographic groups, such as children under 5 years old, indigenous populations, and Afro-Colombians.\n",
            "Description: The dataset contains information related to different sectors and their severity levels in various regions. It includes columns such as admin names, severity codes, and in-need indicators for sectors like CCCM, Education, Health, Nutrition, NFI, Shelter, Food, WASH, ERL, Protection, Child Protection, GBV, and Mine Action. The data provides details on severity levels and in-need indicators for each sector in different regions like Damascus, Aleppo, Ain Al Arab, Al Bab, A'zaz, Jebel Saman, and Afrin.\n",
            "Description: The dataset from the file SYR_FSA_PIN_20233.xlsx contains information on severity levels for the FSA sector in Syria. The data includes severity levels ranging from Stress to Extreme, with corresponding severity codes and descriptions. The dataset also includes columns for Severity, HNO 2023, and an unnamed column. The severity levels are categorized based on a scale where Severity<3 indicates Stress, Severity=3 indicates Severe, and Severity>3 indicates Extreme.\n",
            "Description: The dataset titled \"SDG 4 Global and Thematic data3.csv\" contains information on various indicators, country IDs, years, values, magnitudes, and qualifiers. The data includes indicator codes, country codes, and numerical values for different indicators across multiple years. For example, there are entries for indicator \"ADMI.ENDOFLOWERSEC.MAT\" and \"ADMI.ENDOFLOWERSEC.READ\" for the country with the country code \"ZWE\" from 2014 to 2021. The dataset appears to track specific metrics or indicators over time for different countries.\n",
            "Processed 34.0% resources ...\n",
            "Description: The dataset contains information on attacks on health care during the COVID-19 pandemic in 2020. It includes details such as event number, date, country, COVID-19 event type, conflict event type, perpetrator category, weapons used, and various impacts on health workers such as killings, kidnappings, arrests, threats, injuries, assaults, and sexual violence. The data shows incidents in countries like Sudan, Syria, Yemen, Palestine, Democratic Republic of Congo, and Libya, involving different types of conflict events and perpetrators.\n",
            "Description: The dataset contains information on various departments and their corresponding villages/sites, with data on the number of affected households, total affected population, casualties, destroyed houses, dead animals, destroyed surface area, flooded schools, latrines, health centers, and water points. The data includes details such as the department name, village/site name, and specific values for each category. The dataset provides a comprehensive overview of the impact of flooding in different areas, including the extent of damage and affected populations.\n",
            "Description: The dataset from the file \"SSD_floods_3011223.xlsx\" contains information on flood-affected people in various states and counties in South Sudan. The data includes columns for State P code, County P code, and the assessed number of flood-affected people. The top rows show the column headers, followed by specific data entries for different states and counties, along with the corresponding number of flood-affected individuals. The dataset provides details such as the state and county names, their respective codes, and the number of people affected by floods in each location.\n",
            "Description: The dataset contains information on various towns in Ethiopia, including their names, administrative codes, status as Wereda Capitals, and geographical coordinates. Each town is associated with specific administrative divisions such as regions and zones within Ethiopia. The dataset includes columns for town names (T_NAME), administrative codes (ADMT_PCOD), Wereda Capital status (T_Status), as well as administrative divisions at different levels (ADM3_EN, ADM2_EN, ADM1_EN, ADM0_EN). Geospatial coordinates for each town are also provided in the dataset (POINT_X, POINT_Y).\n",
            "Description: The dataset contains information on various organizations involved in humanitarian and development activities in Nigeria, including their acronyms, types of organizations, operation types, sectors, activities, status, states, LGAs, and response types. The data spans from July to September 2022 and includes details such as the start date and year of operations. Organizations like Achieving Health Nigeria Initiative and Action Against Hunger are highlighted, along with their specific activities related to nutrition, water, sanitation, and hygiene. The dataset also indicates whether the organizations are implementing or reporting on their activities, as well as their involvement in the Humanitarian Response Plan (HRP) and Nexus approaches.\n",
            "Description: The dataset in the file \"ner_hno_pin_20233.xlsx\" contains information on different population groups in various regions, with columns representing different categories such as education, health, nutrition, and protection. Each row corresponds to a specific region and includes data on population statistics for each category. The dataset includes regions like Agadez, Arlit, Bilma, Iferouane, Ingall, Tchirozerine, and Diffa, with corresponding population figures for different categories. The dataset seems to be structured with rows representing regions and columns representing different population groups and categories.\n",
            "Description: The dataset contains information on various regions and districts in Afghanistan affected by winter conditions, specifically focusing on physical access challenges due to road closures and blockages during the winter season. The data includes details such as the region, province code, district code, closure status (high/medium/low), and descriptions of the physical environment related to road status during winter. The dataset highlights the impact of heavy snowfall, poor road networks, and snow clearance efforts on road blockages and movement restrictions in different regions and districts.\n",
            "Description: The dataset contains information on various locations in Libya and their corresponding P_CODE, sector short name, partner, and implementing partner. The activities listed include conducting explosive hazard clearance, providing explosive hazard risk education, and implementing survey and mapping of contaminated areas. The data appears to focus on activities related to Explosive Ordnance Disposal (EOD), EOD spot tasks, Battle Area Clearance, mechanical clearance, and assessments of contamination in different areas.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (3,8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on COVID-19 vaccinations, including data such as location, ISO code, date, total vaccinations, people vaccinated, people fully vaccinated, total boosters, daily vaccination statistics, and various vaccination rates per hundred and per million. The data includes details on different countries and dates, with some entries showing specific vaccination numbers while others have missing values. The dataset provides a comprehensive overview of vaccination progress across different locations and time periods.\n",
            "Description: The dataset contains information on various communes in Madagascar affected by drought, including details such as evaluation date, commune type, region, district, number of households and individuals in the commune, reasons for displacement, destinations of displaced individuals, and returnee information for the years 2019, 2020, 2021, and 2022. The data includes specific figures for each year, such as the number of households and individuals affected, percentages, and reasons for displacement. Additionally, it provides information on migrants in the commune, including details on their destinations and reasons for migration.\n",
            "Processed 36.0% resources ...\n",
            "Description: The dataset contains population estimates for different provinces in Afghanistan, distinguishing between urban and rural populations. It includes columns for Province Code, Province Name, Urban Population, Rural Population, and Total Population. The data shows varying population sizes across provinces, with some provinces having a higher urban population compared to rural, while others have a higher rural population. The dataset provides a comprehensive overview of population distribution in Afghanistan's provinces.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/FieldsData_3W_PHL_Aklan3.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Description: The dataset contains information on the planning of returns for internally displaced persons in Diffa, with columns for village names, total population, and total households. The data includes details such as the number of affected individuals and households in each village. The dataset appears to be structured with rows representing different villages and their corresponding population and household numbers.\n",
            "Description: The dataset contains information on organizations, implementing partners, donors, regions, provinces, and sectors/clusters in West and Central Africa. The data includes columns for Organization, Implementing Partner, Donor, Region, Province, Sector/Cluster, and an additional column labeled Column10. The dataset appears to capture activities related to shelter and AME (Assessment, Monitoring, and Evaluation) in various regions and provinces.\n",
            "Description: The dataset contains information on households and individuals in different states, LGAs (Local Government Areas), and wards. Each row represents data for a specific location, including State Name, State Pcode, LGA Name, LGA Pcode, Ward Name, Ward Pcode, number of households, and number of individuals affected. The data shows the distribution of affected households and individuals across various locations within the ADAMAWA state.\n",
            "Description: The dataset provides information on certified airfields in Ukraine as of 6 July 2022, sourced from the State Aviation Administration of Ukraine. It includes details such as airfield names, airport names, airfield types, and certification statuses. The data emphasizes compliance with Ukrainian aviation rules and restrictions on airfield operation without proper certification. Additionally, it notes the removal of information that could pose risks during military actions by the Russian Federation in Ukraine. The dataset aims to provide a comprehensive overview of airfield metadata while ensuring safety and regulatory compliance.\n",
            "Description: The dataset contains information on operational presence in Afghanistan from July to September 2023, with columns including region, province, district, organization acronym and name, organization type, cluster sector code and name, and a related question. The data shows the presence of organizations like CARE International in various provinces and districts, focusing on sectors such as emergency shelter, non-food items, food security, agriculture, and education in emergencies. The dataset provides a detailed overview of operational activities in different regions of Afghanistan during the specified time period.\n",
            "Description: The dataset contains information from a survey conducted in various wards in South Africa, specifically focusing on baseline assessments. The data includes details such as survey dates, reported dates, province, district municipality, local municipality, ward information, number of households, number of individuals, presence of IDP households, water-related issues, food security challenges, health challenges, infrastructure damage, protection risks, vulnerable groups, priority needs, access to essential services, partner support, and community priority needs. The survey also covers potential solutions to identified problems and the confidence level of the information provided. The dataset highlights various challenges faced by communities, such as water scarcity, food insecurity, health issues, and protection risks, along with proposed solutions and priority needs.\n",
            "Description: The dataset contains information on various regions and districts in Somalia, including data on target population, reach in different sectors (CCCM, Education, FSC, Health, Nutrition, GP, CP, GBV, HLP, Shelter, WASH), intercluster reach, percentage reach, total gap, and percentage gap. The data shows the number of affected individuals, the number reached in different sectors, and the remaining gap in reaching the target population. The dataset provides insights into the response to drought in Somalia from January to June, with details on different regions and their respective needs in various sectors.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Original Google Sheet3.google sheet\n",
            "Description: The dataset titled \"Base_vacantes_laborales_demandadas_ofertas_laborales_migrantes_Enero_Junio3.xlsx\" contains information on labor demand and supply for Venezuelan migrants from July 2020 to June 2022. It includes data on migrants registered in the Public Employment Service by department, types of vacancies (high turnover, neutral, difficult to fill), and labor demand categorized by CIUO 08 AC classification. The geographical breakdown includes national, departmental, and municipal levels. The dataset comprises labor records from 2020 to June 2022, with no specific periodicity mentioned.\n",
            "Description: The dataset contains population statistics for different regions in Somalia for the year 2014. It includes columns for Admin1 Pcode, Admin1 Name, UNFPA 2014 Total population, UNFPA 2014 Urban population, UNFPA 2014 Rural Population, and UNFPA 2014 IDP Population. The data shows the total population, urban population, rural population, and internally displaced persons (IDP) population for each region. The regions listed include Awdal, Woqooyi Galbeed, Togdheer, Sool, Sanaag, Bari, Nugaal, Mudug, and Galgaduud.\n",
            "Processed 38.0% resources ...\n",
            "Description: The dataset contains information on various sectors such as Gender Based Violence, Nutrition, Shelter & Non-Food Items (S&NFI), Water, Sanitation & Hygiene (WASH), Education, Child Protection, Food Security & Livelihoods, Health, General Protection, Mine Action, Refugees, and Total figures for the year 2022. The data includes details on the planned targets and actual reached numbers for each sector in different localities within Sudan. The dataset provides a comprehensive overview of the response and activities carried out in these sectors from January to June 2022.\n",
            "Description: The dataset contains information on various aid activities in Nepal, including details such as activity ID, reporting organizations, start and end dates, descriptions, funding amounts in different currencies, country allocation percentages, sector information, and location details. The activities are implemented by organizations like the United Nations Development Programme, Gavi, The Vaccine Alliance, Drugs for Neglected Diseases initiative, and UN Pooled Funds. The dataset also includes links to additional information for each activity.\n",
            "Description: The dataset contains information on language data for different regions, with columns such as admin0_name, admin0_pcode, number_of_named_languages, main_language, main_language_share, and various language percentages. The data includes details on population totals, gender breakdown, language data availability, data confidence levels, and additional notes. The dataset primarily focuses on language distribution in Somalia, with percentages for various languages spoken in the region.\n",
            "Description: The dataset contains information related to humanitarian coordination in Afghanistan for the months of April, May, and June 2022. It includes details on operational presence (OP) and operational capacity (OC) of organizations working in different districts. The dataset categorizes organizations based on their activities, sectors, and locations. Additionally, it provides information on the total number of unique actors by district with operational presence and operational capacity across all clusters. The dataset also includes sheets listing actor acronyms by district and the unique number of humanitarian actors by cluster and district. Protection actors' names have been intentionally removed for confidentiality reasons, and the dataset is described as a work in progress.\n",
            "Description: The dataset contains information on the Ukraine Flash Appeal for March to December 2022, with data on various clusters such as Camp Coordination, Education, Emergency Telecommunications, Food Security, Health, Logistics, and Multi-Purpose Cash. The data includes the number of people in need, the target number of individuals, and the funding requirements in US dollars for each cluster. The total funding required for all clusters is also provided.\n",
            "Description: The dataset contains information on various organizations involved in humanitarian activities in Nigeria, including their acronyms, types of organizations, operation types, project sectors, activities, status, states, LGAs, and response types. The data includes details such as the provision of cash grants, vocational skills training, and capacity building activities. Organizations like COOPI, IRC, IOM, and IDS are mentioned, along with their respective projects and implementation statuses. The dataset covers activities related to early recovery and livelihoods, with a focus on different regions in Nigeria such as Yobe and Borno.\n",
            "Description: The dataset contains information sourced from EM-DAT, CRED/UCLouvain, Brussels, Belgium, with columns representing various attributes such as country codes, names in different languages, disaster details, event names, start and end dates, and impact metrics like total deaths, injuries, and affected populations. The data includes historical disaster events in Haiti dating back to 1909, categorized by disaster type and subtype such as storms, tropical cyclones, landslides, and floods. The dataset provides a comprehensive record of natural disasters in Haiti over several decades, detailing the associated impacts on the population.\n",
            "Description: The dataset contains information on access to water by commune in Haiti, with columns including administrative codes and names in English, French, and Haitian Creole, population data, and the percentage and number of people with access to water in each commune. The dataset provides details on various communes in the West department of Haiti, such as Port-au-Prince, Delmas, Carrefour, Pétion-Ville, Kenscoff, Gressier, Cité Soleil, Tabarre, and Léogâne, along with their respective population sizes and access to water percentages.\n",
            "Description: The dataset contains information on various sectors such as GBV, nutrition, NFI, wash, education, child protection, FSL, health, protection, mine action, RCF, and total reached for the year 2021. The data includes details like priority codes, administrative divisions, state and locality codes and names, as well as targets and actual figures for each sector from January to December. The dataset provides a breakdown of the reached targets for each sector in different regions within Aj Jazirah, including Abyei PCA, Al Hasahisa, Al Kamlin, Al Manaqil, Al Qurashi, Janub Aj Jazirah, Medani Al Kubra, Sharg Aj Jazirah, and Um Algura.\n",
            "Description: The dataset contains information on the estimated number of people affected by natural disasters, conflict, and disease outbreaks, as well as the number of people in need of emergency response planning in various regions. The data includes columns for state names, the number of people affected by different types of crises, and the planning figures for emergency response in 2022. The dataset covers regions such as Abyei PCA, Aj Jazirah, Blue Nile, Central Darfur, East Darfur, Gedaref, Kassala, and Khartoum, providing insights into the impact of different crises on populations in these areas.\n",
            "Processed 40.0% resources ...\n",
            "Description: The dataset contains information on homicides and feminicides of women in different municipalities within the Antioquia department for the period of January to March 2023. The data includes columns such as Department, Municipality, Divipola code, number of homicides, number of feminicides, total female population, homicide rate, and feminicide rate. The dataset provides a detailed breakdown of these statistics for each municipality, highlighting the varying levels of violence against women across different regions.\n",
            "Description: The dataset contains information on COVID-19 health facilities in Afghanistan by province, including the number of hospitals for COVID-19 patients, testing capacity per day, and the date of the data. The data includes provinces such as Balkh, Herat, Kabul, Kandahar, and Nangarhar, with varying numbers of hospitals and testing capacities. The dataset provides data for different dates, allowing for tracking changes over time in the availability of health facilities for COVID-19 in Afghanistan.\n",
            "Description: The dataset contains information on confirmed COVID-19 cases in Ethiopia, with data recorded from March 13, 2020, onwards. The data includes the number of cases in the last 24 hours, breakdowns by region (such as Addis Ababa, Somali, Oromia, etc.), testing numbers, cumulative totals, patients in treatment centers, patients in intensive care, recoveries, deaths, and percentages. The dataset shows daily updates on new cases, recoveries, and deaths, as well as testing information. Some entries contain missing or incomplete data, denoted by \"NaN\" or \"#DIV/0!\".\n",
            "Description: The dataset contains information on COVID-19 cases in different states of Somalia as of April 27, 2021. It includes data on the number of confirmed cases, deaths, recoveries, and active cases in each state. The states mentioned are Banadir, Galmudug State, Hir-Shabelle State, Jubaland State, Puntland, Somaliland, and South West State. The data shows fluctuations in the number of cases over time, with the most recent data available for April 27, 2021.\n",
            "Description: The dataset contains information on COVID-19 cases in various locations in Libya as of September 24, 2020. It includes columns for location, test samples, confirmed cases, recoveries, deaths, active cases, and date. The data shows the number of test samples, confirmed cases, recoveries, deaths, and active cases for each location such as Tripoli, Benghazi, Misrata, and others. The dataset provides a snapshot of the COVID-19 situation in different regions of Libya on the specified date.\n",
            "Description: The dataset contains information on various host communities in Nigeria, specifically in the Benue region. It includes details such as site locations, demographics, living conditions, access to services like healthcare and education, security, livelihood support, and protection concerns. The data also covers aspects like water and sanitation facilities, food assistance, shelter materials, and the presence of service providers for various needs. Additionally, it captures information on the intentions of displaced persons to return to their places of origin, security incidents, communication preferences, and community dynamics.\n",
            "Description: The dataset contains information about various sites in South Sudan, including details such as location names, nearest villages, state names, GPS coordinates, accessibility, settlement types, IDP displacement reasons, site ownership, community leaders, site management agencies, water sources, sanitation facilities, hygiene practices, food assistance access, livelihood challenges, health concerns, education access, security provisions, conflict resolution mechanisms, security concerns, risk areas, mine presence, community measures, and IDP demographics. The data also includes information on shelter types, materials, NFI needs, cooking fuel sources, lighting sources, energy priorities, water sources, treatment, quality, latrine conditions, garbage disposal, hygiene campaigns, food assistance access, market access, livelihood challenges, health concerns, education access, security provisions, conflict resolution mechanisms, security concerns, risk areas, mine presence, community measures, and IDP demographics.\n",
            "Description: The dataset contains information on COVID-19 cases in different departments of Haiti as of October 8, 2021. It includes columns for the date, department name, cumulative cases, new cases in the last 24 hours, cumulative deaths, new deaths in the last 24 hours, and the case fatality rate. The data is sourced from the Ministry of Public Health and Population of Haiti.\n",
            "Description: The dataset contains information related to child protection projects in the occupied Palestinian territory under the Humanitarian Response Plan 2022. It includes details such as project names, requested funding, target numbers, cluster names, governing entities, and organization names and types. Various international NGOs are involved in providing services such as child protection, mental health, and psychosocial support to vulnerable children and families affected by conflict-related violence and gender-based violence in different regions of the West Bank and Gaza Strip.\n",
            "Description: The dataset contains information on displacement in Nigeria, specifically in the North Central & West regions. It includes data on various factors such as the state, local government area, ward, estimated household numbers, number of IDPs, year of displacement, IDP categories, state of origin, reasons for displacement (insurgency, clashes, disaster, others), previous displacement information, arrival dates of IDP groups, and details on multiple displacements. The data provides insights into the causes and patterns of displacement in the region, with a focus on communal clashes as a significant factor.\n",
            "Processed 41.0% resources ...\n",
            "Description: The dataset contains information on GDP deflator values for Turkey from 1970 to 1978. Each entry includes details such as the ISO3 country code (TUR for Turkey), start and end dates, area code, area name, item code, item name (GDP Deflator), element code, element description, year code, year, unit of measurement (LCU), value, and flag. The GDP deflator values are provided in local currency units (LCU) at 2015 prices, with values ranging from 0.0001 to 0.0005.\n",
            "Description: The dataset contains conflict data for Turkey, with information such as conflict ID, year, start and end years, type of violence, conflict names, dyad names, side A and side B details, number of sources, location coordinates, descriptions, administrative divisions, geographical coordinates, country details, region, event clarity, date precision, start and end dates, death counts for different parties involved, and other related information. The dataset includes multiple entries detailing various conflicts in Turkey, involving different government and rebel groups, along with specific incidents and casualties.\n",
            "Description: The dataset contains information on dengue cases and deaths in the Philippines from 2016 to 2013. It includes columns for location, number of cases, number of deaths, date, and region. The data is structured with rows representing different instances of dengue cases in specific locations, with corresponding case and death counts, dates, and regions. The dataset appears to be organized for analysis and tracking of dengue outbreaks over time and across different regions in the Philippines.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (20,21,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information about various schools, including their names, locations, dates started, pupil populations in 2012 and 2015, ISCED levels, addresses, operators, coordinates, and priority levels. The schools are primarily located in the Autonomous Region in Muslim Mindanao, specifically in Tawi-tawi and Simunul. The dataset includes elementary and high schools, with details such as district affiliations and educational divisions. The dataset seems to focus on schools in a specific region and provides a comprehensive overview of key school attributes.\n",
            "Description: The dataset contains information on IDPs (Internally Displaced Persons) and returnees in Uganda, with details such as snapshot date, survey date, administrative divisions, number of IDP households and individuals, number of returnee households and individuals, area of origin of IDPs, and type of displacement. The data includes various regions in Uganda such as Manafwa, Sironko, Namisindwa, and Bulambuli, and the displacement reasons are primarily natural disasters like floods. The dataset provides a snapshot of the displacement situation in Uganda during the specified survey dates.\n",
            "Description: The dataset contains information on victims of incidents in Colombia from 2017 to 20213. It includes columns such as country, department code and name, year, type of incident, gender, ethnicity, disability status, age range, and number of individuals affected per occurrence. The data provides details on the location, characteristics, and impact of victimizing incidents, including demographic information and categorizations related to ethnicity and disability.\n",
            "Description: The dataset contains information on the number of learners enrolled at different levels of education, including pre-primary, primary, lower-secondary, upper-secondary, and tertiary education levels. The data is based on the latest UNESCO Institute of Statistics data and includes details on school closures and openings in various countries due to the coronavirus pandemic. Some countries have limited data availability for tertiary education enrollment, while others have implemented measures such as online classes and restrictions on face-to-face interactions in educational institutions.\n",
            "Description: The dataset contains information related to population estimates and severity targets for various population groups in Iraq for the year 2022. It includes data on different categories such as In-Camp IDPs, Out-of-Camp IDPs, Returnees, and Overall population figures. The data includes preliminary cluster estimates, population numbers, and various indicators such as children, education, health, and critical indicators. The dataset also provides estimates for acute PIN (Population in Need) and activities requirements for different population groups.\n",
            "Description: The dataset contains information related to a baseline assessment round in Zimbabwe following Cyclone IDAI, including data on affected populations, destroyed homes, IDP households, returnee households, infrastructure damage, and needs assessment. The data includes details such as reported dates, assessment dates, enumerator gender, location information (country, province, district, ward), and various indicators related to cyclone impact and humanitarian needs. Additionally, the dataset captures information on IDP populations, foreigners among IDPs, infrastructure damage types, and the status of various facilities like schools, health facilities, bridges, roads, and public buildings. The dataset also covers details on the residence status of IDPs, including those in camp-like sites, formal camps, or using host family/community accommodation.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Zimbabwe Village Assessment — Matabeleland South and Masvingo Provinces (November 2021)3.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 23000 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Processed 43.0% resources ...\n",
            "Description: The dataset contains information on COVID-19 cases in different regions of Nigeria, with columns including ID, date, ISO code, country name, region name, number of confirmed cases, deaths, recoveries, and gender breakdown of cases. The data is sourced from the Nigeria Centre for Disease Control and covers various regions such as Abia, Adamawa, Akwa Ibom, Anambra, Bauchi, Bayelsa, Benue, Borno, and Cross River. The dataset appears to be structured with semicolons as delimiters and includes specific details for each region's COVID-19 situation.\n",
            "Description: The dataset contains information on COVID-19 cases in Burkina Faso at a subnational level, with columns including ID, DATE, ISO_3, PAYS, ID_PAYS, REGION, ID_REGION, CONTAMINES, DECES, GUERIS, and various vaccination-related data. Each row represents a specific region within Burkina Faso, detailing the number of confirmed cases, deaths, recoveries, and vaccinations. The data is sourced from the Ministry of Health in Burkina Faso and is structured in a CSV format.\n",
            "Description: The dataset contains information on COVID-19 cases in Mauritania at a subnational level, with columns including ID, date, ISO code, country name, region, number of confirmed cases, deaths, recoveries, and gender-specific case counts. The data is sourced from the Ministry of Health and is organized by region within Mauritania. The dataset provides a snapshot of COVID-19 statistics in Mauritania as of March 13, 2020, with zeros reported for all cases at that time.\n",
            "Description: The dataset contains information on COVID-19 cases in Mali, with columns including ID, date, ISO code, country name, region, number of confirmed cases, deaths, recoveries, and gender breakdown of cases. Each row represents data for a specific region within Mali, with details such as the number of infected individuals, deaths, and recoveries.enn.\n",
            "Description: The dataset contains information on various administrative regions in Venezuela, including the number of named indigenous languages, the main indigenous language spoken, its share, and whether the region belongs to an indigenous tribe. It also includes data on the speaking rate of indigenous languages, literacy rates, population demographics, and language data confidence levels. The dataset provides detailed information on indigenous languages spoken in different regions of Venezuela and their corresponding demographic and linguistic characteristics.\n",
            "Description: The dataset contains information on projects related to food security in Colombia for the year 2022. It includes columns such as department code, cluster, project details, partner organizations, department name, and target numbers. Each row represents a specific project with details on the cluster it belongs to, the project name, partner organization, department location, and target number. The projects are focused on emergency humanitarian response and protection of children in various departments of Colombia.\n",
            "Description: The dataset contains information on cases related to sexual exploitation, including details such as stage of legal process, department, municipality, charges, arrests, victim age group, country of birth, and total number of victims. The data includes entries for different stages of legal proceedings, with varying outcomes and characteristics of the victims. The dataset provides insights into the prevalence and characteristics of sexual exploitation cases in different regions, highlighting factors such as age groups of victims and legal outcomes.\n",
            "Description: The dataset contains information on various departments and municipalities in Colombia, including the number of people affected by natural disasters, the population in 2021, rates of affected individuals per 100,000 inhabitants, types of disaster impacts, and the number of victims of presumed sexual crimes. The data also includes the population of women in 2021 and rates of sexual crimes per 100,000 inhabitants. The dataset provides a detailed overview of the impact of natural disasters and sexual crimes in different regions of Colombia, highlighting variations in affected populations and crime rates across different areas.\n",
            "Description: The dataset contains information on different administrative regions in Ukraine, including population demographics and targeted assistance figures. It includes data such as population size, total people in need, total targeted assistance, and breakdowns by age group and gender. The dataset covers regions such as Donetska and Luhanska, providing details on children, adults, elderly, persons with disabilities, women, and men, along with the percentage distribution within each category.\n",
            "Description: The dataset contains information related to projects and activities aimed at providing access to safe and confidential services for survivors of Gender Based Violence (GBV) in vulnerable populations during the COVID-19 pandemic in Colombia. The data includes details such as project IDs, organization names, project descriptions, activity types, estimated costs, start and end dates, status, locations, and the number of individuals reached. The activities involve advocacy, coordination, empowerment, and training to address GBV issues. Additionally, the dataset captures demographic information of the beneficiaries, including girls, boys, women, men, ethnic groups, and individuals with disabilities. The dataset also indicates whether the activities are related to emergencies, armed violence, COVID-19, or natural disasters.\n",
            "Processed 45.0% resources ...\n",
            "Description: The dataset is stored in an Excel file named PAK_Consolidated_4W_Q1_Q43.xlsx and contains information related to various topics such as Capacity Building Workshop on Patient Safety Research, Patient Safety Friendly for Primary Care Framework, National Patient Safety Conference, Advocacy for HRH capacity building, Afghan Refugees, Afghan Refugee Support, Agriculture Support, and Assessment of General Practice of Patient Safety and Adverse Events at Identified Sample Hospitals in Pakistan. The data includes row labels and some information under the Unnamed: 0 column.\n",
            "Description: The dataset contains information on COVID-19 cases in various provinces of Mozambique as of February 13, 2022. It includes data on the number of positive cases, deaths, and recoveries in each province. The provinces listed include Cabo Delgado, Niassa, Nampula, Zambézia, Tete, Manica, Sofala, Inhambane, and Gaza. The dataset provides a snapshot of the COVID-19 situation in these provinces on the specified date, with varying numbers of cases, deaths, and recoveries reported across the different regions.\n",
            "Description: The dataset contains information on different departments in Colombia, including the department name, code, number of people in need, targeted population, number of beneficiaries reached, reach percentage, type of reach (sub-achieved, over-achieved, or no beneficiaries reported), and various indicators (ind_1 to ind_6). Some departments have sub-achieved their targets, while others have over-achieved. Some departments have no beneficiaries reported. The dataset provides a snapshot of the progress and challenges in reaching the targeted populations in different regions of Colombia.\n",
            "Description: The dataset titled \"Government Actions on COVID-19 in Developing Countries\" contains information on national policies and actions in response to the COVID-19 pandemic, focusing on developing countries. It includes data for 20 Global South countries and 6 Global North countries, sourced from publicly available information and Dalberg staff's on-the-ground knowledge. The dataset covers 100 non-pharmaceutical interventions categorized into health-related, policy-related, distancing and hygiene, and economic measures. Additionally, the dataset provides contact information for submitting comments and questions to Dalberg.\n",
            "Description: The dataset contains information on displacement due to Tropical Storm Ana in Malawi, with columns including Snapshot Date, Country, Region, District, Traditional Area, Location Name, and Total Displaced Households. The data shows the date of reporting, country code, region code, district code, traditional area code, and the number of displaced households in various locations such as schools and camps. The dataset provides insights into the impact of the storm on displacement in different areas of Malawi.\n",
            "Description: The dataset contains information on various indicators related to health and population demographics for different regions in Colombia. It includes data such as department codes, municipality names, total population, healthcare indicators, mortality rates, severity clusters, and gender-specific population numbers. The dataset also includes information on factors like conflict, morbidity rates, and various mortality rates per 100,000 population for different diseases. The dataset covers multiple regions within the Antioquia department and provides detailed insights into the health and demographic characteristics of these areas.\n",
            "Description: The dataset contains information on education cluster activities in different governorates in Yemen. It includes columns such as Governorate, Districts, Activity, Sub-Activity, and details on various educational initiatives such as school rehabilitation, provision of school desks, supplies, teacher training, and hygiene education. Each row represents a specific activity in a particular district, along with associated codes and details. The dataset also includes information on the type of support provided, such as establishing temporary learning spaces, providing school bags, textbooks, and supporting national exams.\n",
            "Description: The dataset from the file Zim_3W_August_20213.xlsx contains information on activities in Zimbabwe for August 2021. It includes details such as reporting month, province, areas, sector/cluster, lead agency, implementing partner, and organization type. The data shows activities related to education and protection/child protection, with organizations like World Vision International and United Nations Children's Fund being involved. The dataset seems to focus on humanitarian efforts and partnerships in Matabeleland South, particularly in Beitbridge, involving both international and non-governmental organizations.\n",
            "Description: The dataset is stored in an Excel file and contains metadata related to indicators of deprivations and deficiencies in access to public services by refugees, migrants, and returnees from Venezuela and host communities in Colombia. The data pertains to the WASH sector and is sourced from the National Administrative Department of Statistics (DANE). The dataset, named \"200402-datos-dashboard-acceso-a-servicios,\" is based on secondary information and aims to understand the access conditions to public services for the specified population groups. The main variables include gender, area, department, age group, variables, sector, and department name.\n",
            "Description: The dataset is stored in an Excel file and contains metadata related to water access, sanitation, and solid waste collection for the national population in Colombia based on the 2018 census. The data falls under the WASH sector/theme and originates from the Department of National Statistics (DANE). The dataset is named \"datos-para-hpc-v2\" and is a secondary information source. Its purpose is to understand the conditions of water access, sanitation, and waste collection in Colombia, with key variables including Divipola codes by department, municipality, national population, and access to water, sanitation, and waste collection.\n",
            "Processed 47.0% resources ...\n",
            "Description: The dataset contains information on IRCA (Índice de Riesgo de Calidad del Agua) values for different municipalities in Antioquia from the years 2010 to 2019. The data includes columns for the year, department, municipality, IRCA value, and category of risk associated with the water quality. The dataset provides a snapshot of the water quality status in various municipalities over the specified time period, with some municipalities categorized as \"Sin riesgo\" (No risk), \"Medio\" (Medium risk), or \"Bajo\" (Low risk) based on their IRCA values.\n",
            "Description: The dataset contains information on WASH indicators for foreign migrants who have moved less than 5 years ago. The data includes details such as municipality codes, total population, access to sanitary facilities, access to aqueducts, access to sewer systems, and access to trash collection services. The dataset provides specific figures for each municipality, including the number of foreigners with access to these services and the corresponding percentages of those without access.\n",
            "Description: The dataset contains information on the coverage of sewerage, waste disposal, and water supply services in Colombia. It includes variables such as region, department, area classification, total households, and percentages of coverage for water supply, sewerage, and waste disposal services in 2018. The data is disaggregated geographically at the national, departmental, municipal, and rural levels. The dataset is sourced from the National Quality of Life Survey conducted by DANE in 2018.\n",
            "Description: The dataset contains information on the number of Afghan refugees returning to Afghanistan between January 1st and December 31st, 2021, as reported by UNHCR data collected at the Encashment (returnee) center. The data includes columns such as Country of Asylum, Return Date, Province of Origin, District of Origin, Province of Destination, and District of Destination. The dataset is stored in an Excel file located at the specified path.\n",
            "Description: The dataset titled \"Base convalidaciones - HDX3.xlsx\" contains information on individuals who have validated their academic titles in Colombia, originally obtained in Venezuela. The data includes details on whether these validations were approved or rejected. The dataset provides information on variables such as academic titles, academic institutions, geographical breakdown at national, departmental, and municipal levels, as well as population segments that have validated their titles. The data spans from 2015 to 2021 and is updated as needed.\n",
            "Description: The dataset contains information on school dropout rates in different regions, with columns representing various data points such as department code, department name, municipality code, municipality name, official school counts, privacy school counts, total enrollment, dropout counts, and dropout percentages for both official and private schools. The data includes details such as the number of students enrolled, dropout counts, and percentages for primary, secondary, and media education levels in different regions. The dataset provides a comprehensive overview of dropout rates in educational institutions across various areas.\n",
            "Description: The dataset contains information on internet connectivity for different municipalities in the Amazonas department for the year 2018, broken down by trimester. It includes data such as the number of individuals without fixed internet access, the population aged 5 to 19, and an index of connectivity. The dataset provides insights into the digital divide and connectivity challenges in the region, with varying levels of internet access across different municipalities.\n",
            "Description: The dataset contains information on educational coverage in different locations, including department and municipality codes, student enrollment numbers for primary, secondary, and high school levels, total student population, age distribution, and coverage percentage. The data includes details such as the number of students in different age groups, total student counts, and coverage percentages for various municipalities within Antioquia, Colombia.\n",
            "Description: The dataset contains information about healthcare institutions in Colombia, including details such as the department, municipality, provider code, provider name, habilitation code, address, neighborhood, nature of the institution, level, and other related information. Each entry represents a different healthcare institution, with data on whether it is habilitated, its classification, and other relevant details. The dataset provides a comprehensive overview of various healthcare facilities in different regions of Colombia.\n",
            "Description: The dataset contains information about various organizations and their activities in different regions related to cooperation for migrant children and adolescents. It includes details such as the names of organizations, their acronyms, and their presence in different territories. The data also covers interventions in areas like mental health, nutrition, violence prevention, disability inclusion, substance abuse prevention, and more. Each organization's involvement in these interventions across different regions is outlined, providing a comprehensive overview of their contributions to various aspects of support and assistance for migrant populations.\n",
            "Processed 49.0% resources ...\n",
            "Description: The dataset contains information on different departments, municipalities, and organizations in Colombia. It includes columns for \"Departamento\" (Department), \"Municipio\" (Municipality), and \"Organización\" (Organization). The data shows the names of departments and municipalities, along with the organizations operating in each municipality. The dataset provides a snapshot of the presence of various organizations across different regions in Colombia.\n",
            "Description: The dataset contains information on sexual crimes in Colombia, with details such as the year, date, department, municipality, day, time, neighborhood, zone, type of site, weapon used, aggressor and victim's mobility, age, gender, marital status, country of birth, employment class, profession, education level, DANE code, and quantity. The data includes incidents from various departments and municipalities in Colombia, with information on the characteristics of the crimes and the individuals involved.\n",
            "Description: The dataset contains information on support provided by different organizations during the COVID-19 pandemic, including details such as the department, municipality, agency/organization, institution where the support is delivered, type of support, quantity or number of people served, availability status, delivery status, and date of delivery. The data includes entries from various departments and municipalities in Colombia, with support being provided to different entities such as shelters, communities, hospitals, and public institutions. The dataset appears to track the distribution of supplies or aid to address health needs related to COVID-19.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Unknown extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset provided is in an Excel file format and contains information related to the density of HIV/AIDS incidence per 1000 cases in different municipalities of Colombia, categorized by sex. The data includes columns for country, department, municipality, sex, number of cases, and density of incidence per 1000 cases. It appears to have details on various municipalities within the Antioquia department, with specific information on the number of affected individuals and the corresponding density of incidence. The dataset also includes a note mentioning that only live cases for the year 2017 were considered.\n",
            "Description: The dataset SECOP_HDX3.xlsx contains information on public procurement contracts related to the Venezuelan refugee and migrant population. It includes variables describing entities entering into these contracts, with data available at the national, departmental, and in some cases municipal levels. The dataset covers the period from 2005 to 2021 and is updated as needed.\n",
            "Description: The dataset contains information on severity levels in different districts of Iraq in 2021, including Governorate Name, District Name, District Code, Governorate Code, Overall Severity, In-camp IDPs Severity, Out-of-camp IDP Severity, and Returnee Severity. The severity levels range from 0 to 4, with higher numbers indicating greater severity. The data shows variations in severity levels across different districts, with some districts having higher levels of severity for in-camp IDPs, out-of-camp IDPs, and returnees.\n",
            "Description: The dataset contains information on various indicators and demographic data for different municipalities in Colombia. It includes details such as population, birth statistics, food security, malnutrition, disasters, poverty, rurality, and other socio-economic factors. The data is organized by department, municipality, and specific indicators for each location. Additionally, it provides information on the population breakdown by gender, age group, ethnicity, and vulnerability status. The dataset covers multiple municipalities in the Antioquia department and includes data on factors like low birth weight, conflict-affected areas, and the Human Development Index (IPM).\n",
            "Description: The dataset contains information on target populations for health interventions in various municipalities in Antioquia, Colombia. It includes data on the population code, administrative divisions, target population size, severity level, and demographic breakdown by gender and age group. The dataset provides details on the number of men, women, boys, and girls in each municipality, allowing for analysis and planning of health programs tailored to specific population needs.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Proyectos del Clúster de Salud_2022_HDX3.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 3\n",
            "\n",
            "Description: The dataset titled \"Pakistan IDPs by Area of Origin3.xlsx\" contains information on the total number of internally displaced persons (IDPs) in Pakistan by areas of origin. The data includes columns for the serial number, district name, Pcodes, IDP families (assuming a family size of 6), and IDP individuals. The dataset lists the top IDP-affected districts such as North Waziristan, Khyber, Kurram, Mohmand, Orakzai, Tank, South Waziristan, and Bajaur, along with the corresponding number of IDP families and individuals in each district.\n",
            "Description: The dataset contains information on different population groups such as IDPs, Migrants, Non-displaced, Refugees, Returnees, and Grand Total, with data on the number of individuals in various age and gender categories. The dataset includes totals for males and females in different age groups, as well as a grand total for each population group. The data is presented in a tabular format with columns representing different categories and rows representing the population groups.\n",
            "Processed 50.0% resources ...\n",
            "Description: The dataset contains information on various indicators related to water, sanitation, waste collection, disasters, and vulnerable populations in different municipalities in Antioquia, Colombia. The data includes population figures, severity levels for different indicators, disaster rates, and demographic breakdowns by gender, age group, rural population, indigenous communities, Afro-Colombians, and people with disabilities. The dataset also includes a severity score for the wash sector, a cluster score for those in need, and a ponderator value. The data provides a comprehensive overview of the water, sanitation, and hygiene situation in the region, as well as the vulnerabilities and needs of different population groups.\n",
            "Description: The dataset contains information on different areas in Libya, including the population groups categorized by gender and age range. It includes data on the number of individuals in need within each group, such as females and males below 18, between 18-59, and above 60 years old. The data also specifies the affected areas and the type of population group, such as internally displaced persons (IDPs). The dataset provides a detailed breakdown of the population demographics and needs across various regions in Libya.\n",
            "Description: The dataset contains information on various organizations and their funding activities related to health in Colombia. It includes details such as the destination organization, description of the project, funding amount in US dollars, funding status, original amount in the original currency, and the organization's cluster and funding type. Organizations like Médecins du Monde France, Save the Children, and International Planned Parenthood Federation are involved in providing health-related assistance, including humanitarian response and health services to communities affected by crises. The dataset also includes information on funding commitments, paid contributions, and pledges from organizations like Development and Peace, ACT Alliance, and the Pan-American Health Organization.\n",
            "Description: The dataset contains population data for different administrative regions in Ukraine, including the adm1_name, adm1_code, GCA/NGCA, Zone, population groups, and baseline population numbers for children, adults, elderly, Persons with Disabilities (PwD), women, and men. The data includes the total population and percentages for each population group in each region. The dataset provides detailed demographic information for regions such as Donetska and Luhanska, including the breakdown of population by age and gender.\n",
            "Description: The dataset from the file \"Kerela3.xlsx\" contains information about different organizations, their activities, and locations in Kerala, India. The data includes timestamps, source names, organization names, types of organizations, regions, countries, provinces, and sectors of focus. The organizations mentioned are Totem Resource Centre and Neethi Vedhi, engaging in various sectors such as education, health & nutrition, livelihoods, environment protection, food security, and protection. The data seems to capture activities related to social welfare and development in the specified region.\n",
            "Description: The dataset contains weekly data for the year 2021 on unique users on hdx&tools, with columns for Year, Week, Start_date, End_Date, Indicator, and Count. The data includes the number of unique users for each week, starting from Week 1 to Week 9. The Count column shows the count of unique users for each week.\n",
            "Description: The dataset from the file \"PAK_HNO_20213.xlsx\" contains information on different areas in Pakistan, including Balochistan, Islamabad, Khyber Pakhtunkhwa, Punjab, and Sindh. The data includes columns for Area, Total Population, and Total PIN (Population in Need). The dataset provides population figures for each area, with Balochistan having 12.3 million people and 2.1 million in need, Islamabad with 2 million people and 33,000 in need, Khyber Pakhtunkhwa with 35.5 million people and 3.8 million in need, Punjab with 110 million people and 370,900 in need, and Sindh with 47.9 million people and 4.6 million in need.\n",
            "Description: The dataset contains subnational COVID-19 data for various governorates in Iraq as of November 23, 2022. It includes columns for Governorate, Cases, Deaths, Recoveries, Active Cases, and Date. The data shows the number of cases, deaths, recoveries, and active cases for each governorate on the specified date. The dataset provides detailed information on COVID-19 statistics at a subnational level in Iraq.\n",
            "Description: The dataset contains information on disability data in different municipalities within the Antioquia department. It includes columns for department code, department name, municipality name, Divipola code, gender, type of disability, value, population, and disability rate per 100,000 inhabitants. The data shows the number of individuals with mobility disabilities in each municipality, along with their corresponding population and disability rates.\n",
            "Description: The dataset contains information on various activities related to humanitarian aid and development projects in different regions of Colombia. It includes details such as the type of activity, organization leading the implementation, number of beneficiaries, focus on vulnerable groups, and indicators being measured. The data covers a range of topics including food security, nutrition, and crisis response, with a focus on supporting communities in crisis situations. The dataset also includes information on whether the activities are related to COVID-19, armed conflict, natural disasters, or emergency situations.\n",
            "Processed 52.0% resources ...\n",
            "Description: The dataset contains information about schools in Harare, including their school number, name, province, school level, district, grant class, latitude, and longitude coordinates. The schools are primarily located in districts such as Northern Central, Warren Park Mabelreign, Mbare Hatfield, and Mabvuku Tafara. The dataset provides a comprehensive list of schools with their corresponding geographical coordinates, making it useful for mapping and analysis purposes.\n",
            "Description: The dataset contains information about different organizations involved in humanitarian response and peacebuilding activities in various departments of Colombia. Each entry includes the organization's name, department, website, focus areas, target populations, differential target populations, work areas, and types of activities provided. The organizations listed include \"Acción Contra el Hambre\" operating in different regions such as Cundinamarca, La Guajira, Norte de Santander, Santander, Amazonas, Arauca, Caquetá, Vichada, and Putumayo. They focus on providing assistance to refugee and migrant populations, returnees, ex-combatants, and communities affected by armed conflict, natural disasters, and COVID-19, offering services like information and orientation, psychosocial support, first aid, technical assistance, emergency vouchers, health care, and livelihood projects.\n",
            "Description: The dataset contains information on the number of births to mothers aged 10-14, 15-19, and the overall birth rate per 1000 women in different municipalities within Antioquia, Colombia for the year 2020. The data includes columns for department and municipality codes, names, and specific indicators for each age group. The summary shows the distribution of births across different municipalities, with varying numbers of births in each age group.\n",
            "Description: The dataset from the file \"PiN_VBG_HDX3.xlsx\" contains information on different municipalities in Antioquia, Colombia. It includes columns for the department, Divipola code, municipality name, female population, severity level, percentage of females in need, and the Population in Need (PIN) metric. The data shows details such as the female population, severity level, and PIN for each municipality. The dataset can be summarized as a collection of demographic and social vulnerability indicators for various areas within Antioquia.\n",
            "Description: The dataset contains information on different administrative regions in Antioquia, Colombia, including population numbers for 2019 and 2020, number of affected individuals in 2019 and 2020, average population, total affected individuals, and the affected rate. The data shows variations in population and affected individuals across different regions, with some areas experiencing higher rates of impact from disasters compared to others.\n",
            "Description: The dataset contains information on health facilities, including their IDs, location details (province, district, longitude, latitude), facility characteristics (elevation, ownership, year built, type of facility), and various counts (number of doctors, nurses, beds, etc.). The data also includes information on facility amenities and services (water, electricity, sanitation, security, road access, etc.). The dataset seems to focus on health facilities in Matebeleland South, specifically in Beitbridge, with a mix of district hospitals, clinics, and rural health centers.\n",
            "Description: The dataset contains information on reported cases of sexual crimes by gender and age group in different municipalities. The data includes columns for department, municipality, divipola code, gender, age group, and number of cases. The dataset shows that there are missing values for some age groups and genders, while others have reported cases. The dataset appears to be structured for analysis and monitoring of sexual crimes across different regions.\n",
            "Description: The dataset contains information on COVID-19 cases in Burkina Faso at the city level. It includes columns such as ID, DATE, ISO_3, PAYS, REGION, VILLES, COMMUNES_TYPE, Contaminés, Décès, Guéris, Femme, Homme, Genre_non spécifié, and Source. Each row represents data for a specific date and location within Burkina Faso, detailing the number of confirmed cases, deaths, recoveries, and gender distribution. The data is sourced from the Ministry of Health in Burkina Faso.\n",
            "Description: The dataset contains information on COVID-19 cases in Mali, with columns representing ID, date, ISO code, country, region, city, number of confirmed cases, deaths, recoveries, gender breakdown, and data source. The data includes details such as date, location, and number of cases for different regions and cities within Mali. The dataset seems to be structured with semicolons as separators and includes information from various sources like humanitarian responses and the Ministry of Health.\n",
            "Description: The dataset contains information on various camps in Cox's Bazar, including camp IDs, names, settlement types, district, upazila, union, geo codes, total households, total individuals, latitude, and longitude. The camps are categorized as collective sites in the Palong Khali area. The dataset provides details such as the number of households, individuals, and geographical coordinates for each camp.\n",
            "Processed 54.0% resources ...\n",
            "Description: The dataset contains population data for Iraq from 2015 to 2023, with the total population figures for each year. The population numbers increase steadily each year, starting from 35,212,600 in 2015 and reaching 43,324,000 in 2023. The data is stored in an Excel file named \"Iraq_Population_2021_CSO_Projection3.xlsx\" located in the specified directory.\n",
            "Description: The dataset contains information from a baseline assessment round in Mauritania regarding migrants, their demographics, destinations, activities, and needs. It includes data on the number of migrants by gender and age group, their destinations (such as other countries or return to their home country), common activities in the community, and primary needs identified by the migrants. The dataset also captures details on the presence of migrants in different regions and their interactions with the host community.\n",
            "Description: The dataset contains information related to site assessments in Haiti following an earthquake in 2021. It includes details such as the site name, assessment date, site status, reason for displacement, site type, GPS coordinates, administrative information, number of households and individuals affected, and the type of disaster. The data is structured in rows with each row representing a different site assessment. The dataset provides insights into various locations, their conditions, and the impact of the earthquake on the population in terms of displacement and shelter needs.\n",
            "Description: The dataset contains information on estimates of internally displaced persons (IDPs) in Mali from January 2013, including the number of households and individuals in different regions such as Bamako, Gao, Kayes, Kidal, and Koulikoro. The data includes details on the number of men and women, average household size, percentage, data sources, and dates of the estimates. The dataset also indicates whether the information is updated or not for each region.\n",
            "Description: The dataset contains information on security incidents in Ukraine from January to September 2017. Each entry includes the date of the incident, the event type (security incidents), and the number of affected individuals. The data shows varying numbers of incidents each month, with the highest number of affected individuals recorded in March 2017 (2289) and the lowest in July 2017 (838).\n",
            "Description: The dataset contains information on language and demographic data for El Salvador, with columns such as admin0_name, admin0_pcode, number_of_named_languages_besides_spanish, main_indigenous_language, main_language_besides_spanish, population totals, literacy rates, and metadata details. The data includes details on indigenous languages spoken, the percentage of the population that is literate, and confidence levels in the data. The dataset seems to have missing values and metadata information in some rows.\n",
            "Description: The dataset from the file IRV-20213.xlsx contains information on different departments (dpto) and municipalities (mpio) in Antioquia, Colombia. It includes columns such as department code (codigo_dpto), department name (dpto), municipality code (codigo_dt), municipality name (dt), PDET code (codigo_pdet), PDET name (pdet), divipola code (divipola), estimated value (estimado), and cluster classification (cluster). The data provides estimations and classifications for various municipalities within Antioquia, ranging from low to high levels.\n",
            "Description: The dataset contains information on organizations involved in humanitarian activities in Nigeria, with details such as organization acronym, type, operation type, project sector, activities, status, states, LGAs, and response type. The data includes organizations like Action Aid (AA) implementing and reporting on various projects related to sectors like nutrition, early recovery, livelihoods, and governance restoration in states like Yobe and Borno. The dataset covers activities from August 2021 and includes information on whether the projects are ongoing or completed.\n",
            "Description: The dataset contains information on displacements in Somalia due to various reasons such as drought and conflict/insecurity. It includes details such as the year, month, region, district, category of reason for displacement, total number of affected people, and specific dates. The data is structured with columns for year, month number, month name, region, district, category reason, total number of people affected, and date information. The dataset seems to focus on tracking displacements in Somalia and categorizing them based on different factors.\n",
            "Description: The dataset contains information about various organizations, their acronyms, types of organizations, operation types, project sectors, activities, status, states, state codes, LGAs, LGA codes, HRP status, response types, reporting/implementing status, month, and year. The data includes details such as organization names, types, and activities related to nutrition, early recovery, livelihoods, and governance in states like Yobe and Borno in Nigeria. The dataset seems to focus on humanitarian efforts and projects implemented by different organizations in specific regions during August 2021.\n",
            "Processed 56.00000000000001% resources ...\n",
            "Description: The dataset contains information on civilian casualties in Ukraine from 2016 to 20213, sourced from the Office of the High Commissioner for Human Rights (OHCHR). The data includes details on the methodology used to document civilian casualties, which involves consulting various sources and types of information to ensure credibility and reliability. The dataset categorizes casualties by different causes such as shelling, small arms and light weapons fire, explosive remnants of war, and mines. It also notes that conclusions on civilian casualties may be revised as more information becomes available, leading to potential changes in the reported numbers over time.\n",
            "Description: The dataset contains information on hospitals and clinics in Nigeria, including details such as state, local government area (LGA), ward, unique identifier (UID), facility code, facility name, registration number, start date, ownership, facility level, longitude, latitude, operation status, registration status, license status, creation date, and last update date. The data includes various facilities, both public and private, with information on their operational status, registration, and licensing.\n",
            "Description: The dataset provided is stored in an Excel file and contains information related to agriculture clusters in Ethiopia from March to June 2021. The data includes details such as the month of implementation, donors, reporting organizations, implementing partners, administrative divisions, funding sources, objectives, activities, response timing, indicators, activity status, delivery modalities, beneficiaries' details, cluster names, countries, and the number of beneficiaries reached. The dataset seems to focus on various activities aimed at enhancing emergency food production and providing assistance in agriculture, particularly in response to drought situations, through initiatives like capacity building, distribution of agricultural inputs, and cash transfers to different beneficiary groups such as host communities and internally displaced persons.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-27-444004c988ae>:66: DtypeWarning: Columns (3,4,5,7,8,9) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  df = pd.read_csv(resource_name)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on excess mortality during the COVID-19 pandemic, with columns including country, region, period, year, month, week, date, deaths, expected deaths, excess deaths, and total excess deaths percentage. The data shows the number of deaths reported for each month in different countries, along with the expected deaths and excess deaths. The total excess deaths percentage is also provided. The dataset appears to be structured for analyzing and comparing mortality rates during the pandemic across various regions and time periods.\n",
            "Description: The dataset contains daily updates on COVID-19 cases in Indonesia, with columns for Date, Cumulative_cases, Recovered_cases, Total_death, Patient_under_treatment, New_case_perDay, Recovered-cases_perDay, Death_cases_perDay, and Treatment_cases_perDay. The data starts from March 2, 2020, and includes information on the number of cumulative cases, recoveries, deaths, patients under treatment, and daily changes in these metrics. The dataset shows a progression of COVID-19 cases over time, with varying numbers of new cases, recoveries, deaths, and patients under treatment reported each day.\n",
            "Description: The dataset contains information on various Entry-Exit Control Points (EECPs) in Ukraine, including details such as the type of EECP, name, operational status as of July 2021, administrative code, administrative name, latitude, and longitude. The data includes EECPs that are non-operational, operational, partially operational, and planned. Each entry specifies the type of EECP, its name, and its current operational status, along with geographical coordinates. The dataset provides a snapshot of the status of different EECPs in Ukraine as of July 2021.\n",
            "Description: The dataset contains information on incidents affecting the healthcare system in the Central African Republic, including details such as attack ID, date and time of the attack, country/territory, certainty level, attack type, total injured and total deaths, as well as various categories related to healthcare facilities, personnel, supplies, and patients. The incidents involve different types of violence and threats, including psychological violence, removal of healthcare assets, and militarization of healthcare assets. The dataset also includes information on the type of facility affected and specific details about the incidents, such as abduction, arrest, and detention of healthcare workers and patients.\n",
            "Description: The dataset contains information on COVID-19 cases in different regions of the Democratic Republic of Congo. It includes data such as the date, region, number of confirmed cases, deaths, recoveries, gender distribution of cases, and vaccination statistics. Each row represents a specific region within the country and provides details on the mentioned variables. The dataset seems to be sourced from the World Health Organization (OMS RDC).\n",
            "Description: The dataset from the file \"Haiti 3W data3.xlsx\" contains information on organizations in Haiti, including their acronyms and names in both French and English, as well as the departments and communes they are associated with. The data includes entries for various organizations across different regions in Haiti.\n",
            "Description: The dataset contains information on various organizations involved in humanitarian activities in Nigeria, including their acronyms, types of organizations, operation types, project sectors, activities, and status. The data also includes details on the states, LGAs, and response types for each entry. The organizations listed, such as Action Against Hunger (AAH/ACF), are international NGOs reporting on health activities in different regions of Nigeria. The dataset covers activities completed in states like Yobe and Borno during the month of May.\n",
            "Processed 57.99999999999999% resources ...\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM CAR Site Assessment Round 123.xlsx contains information related to DTM Rn12 BD Sites. It provides an overview of PDI estimation by location at the village level. The data includes fields such as _ADM1_NAME, _ADM1_CODE, _ADM2_NAME, _ADM2_CODE, and _ADM3_NAME, which reference administrative levels such as Prefecture, admin1 Pcod, Sous-Prefecture, admin2 Pcod, and Commune/Arrondissement, respectively. The dataset also includes a data dictionary.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset provided is in an Excel file format and contains information related to various fields such as reporting week, status of response, start and end dates, organizations involved, locations, sector/cluster activities, monitoring indicators, quantities, and demographics. The data includes details about different activities conducted by the International Organization for Migration (IOM) in Tigray, specifically related to health services like mobile health and nutrition teams. The dataset also includes information on the number of consultations, cases receiving mental health support, children treated for severe acute malnutrition, and women receiving contraceptives. The data seems to focus on ongoing activities and services provided in specific regions within Tigray.\n",
            "Description: The dataset EC_lang_admin03.csv contains information on the number of named languages, main language, language shares, population demographics, literacy rates, and metadata for Ecuador. The data includes details such as the names and codes of administrative regions, the number of named languages, the main language spoken, its share in the population, and the percentage of literacy among males and females. The dataset also includes information on the total population, gender distribution, and metadata confidence levels.\n",
            "Description: The dataset contains information on humanitarian response activities related to COVID-19, including details such as lead organization, implementing partners, cluster/sector, sub-cluster, region, province, city/municipality, activity description, health pillar (for health-related activities), status (ongoing, completed, planned), start and end dates, as well as region, province, and city/municipality codes. The activities range from hygiene kits distribution, cash distribution, radio program sponsoring, installation of handwashing facilities, emergency cash transfers, to provision of hygiene kits and personal protective equipment. The dataset includes data from various regions in the Philippines and provides a comprehensive overview of the COVID-19 response efforts undertaken by different organizations.\n",
            "Description: The dataset contains information on Sudan Floods Affected Localities as of 10th November 2020. It includes columns such as Shape_Area, ADM2_PCODE, ADM1_EN, ADM2_EN, Houses completely damaged, Houses partially damaged, number of households affected, number of people affected, and number of deaths. The data shows details for various localities in Sudan, including the extent of damage to houses, the impact on households and individuals, and the number of deaths reported.\n",
            "Description: The dataset from the file HTI_TARGET_HRP20213.xlsx contains information on targeted populations in different communes within the Ouest department of Haiti. The data includes details such as the number of women, men, girls, and boys targeted for assistance in each commune, along with corresponding administrative codes. The dataset provides a breakdown of targeted populations by gender and age group, highlighting the varying needs across different areas within the Ouest department.\n",
            "Description: The dataset titled \"Intersectoral severity of needs - Ethiopia 2021 HNO3\" contains information on administrative divisions in Ethiopia, including admin1Name, admin1Pcode, admin2Name, admin2Pcode, admin3Name, admin3Pcode, Severity of General Non-displaced Population, Severity of IDPs Population, Severity of IDP returnee Population, and a disclaimer column. The severity levels for different populations in various regions are provided, with some regions having insufficient data. The dataset appears to be focused on assessing the severity of needs for different population groups in Ethiopia.\n",
            "Description: The dataset contains information on various states and localities in Sudan, including population projections, gender distribution, age demographics, displacement figures, and other demographic indicators. It includes data on different categories such as children under 5 years, women and girls aged 15-49 years, elderly individuals over 65, people with disabilities, internally displaced persons (IDPs), returnees, refugees, vulnerable residents, and various other metrics related to health, nutrition, education, and protection. The dataset provides detailed statistics for each state and locality, including projections for different indicators up to the year 2021.\n",
            "Description: The dataset contains information on the number of named languages, main languages, and their respective shares in a specific administrative region. It includes data on various languages spoken in the region, along with population demographics such as total population, male and female populations, literacy rates, and data confidence levels. The dataset also provides details on language data availability, confidence levels, and additional notes. The dataset pertains to language and demographic information for a specific administrative region, with a focus on language diversity and population characteristics.\n",
            "Description: The dataset provided is from a file named \"UA-IDPs (2021 HNO)3.xlsx\" and contains information related to internally displaced persons (IDPs) in Ukraine. The data includes an explanatory note stating that the figures represent the baseline number of IDPs residing in government-controlled areas in 2020, endorsed by OCHA and other humanitarian partners for the Humanitarian Needs Overview for Ukraine in 2021. The figure is based on the number of IDPs registered by the Government of Ukraine, which is reported to be 1.5 million.\n",
            "Processed 59.0% resources ...\n",
            "Description: The dataset contains COVID-19 city-level data for Togo, with columns including ID, date, ISO_3 country code, administrative region names, new infected, deceased, and recovered cases, as well as gender-specific infection counts. The data is sourced from the official government website. The dataset provides a detailed breakdown of COVID-19 cases by date and region within Togo, including information on gender distribution among the affected individuals.\n",
            "Description: The dataset contains information on COVID-19 cases in Togo at a subnational level, with columns including ID, DATE, ISO_3, PAYS, ID_PAYS, REGION, ID_REGION, CONTAMINES, DECES, GUERIS, CONTAMINES_FEMME, CONTAMINES_HOMME, CONTAMINES_GENRE_NON_SPECIFIE, and SOURCE. Each row represents data for a specific region within Togo, detailing the number of confirmed cases, deaths, recoveries, and gender-specific cases. The data is sourced from the official government website.\n",
            "Description: The dataset contains information on the prevalence of malnutrition among children under five in Pakistan, broken down by locality (urban vs. rural) and gender (boys vs. girls). The data includes indicators such as stunted, wasted, underweight, and overweight, with corresponding prevalence rates for each category. The prevalence rates vary between urban and rural areas, as well as between boys and girls. The dataset is structured in a tabular format with columns representing different categories and subcategories of malnutrition indicators.\n",
            "Description: The dataset contains information on various regions in Zimbabwe, including population projections and demographic breakdowns. It includes data on different population groups such as residents, IDPs (Internally Displaced Persons), returnee migrants, and refugees. The dataset also provides details on gender distribution, age groups, and percentages for males, females, children, boys, girls, men, women, adults, and elderly individuals. The data is structured with columns for different administrative regions, population projections, and demographic indicators.\n",
            "Description: The dataset contains information on operational presence in various regions, provinces, and districts of Afghanistan from January to March 2021. It includes details such as organization acronyms and names, organization types, cluster sector codes and names, and specific questions related to operational presence. The data showcases the presence of different national NGOs in sectors like emergency shelter, non-food items, nutrition, and health across different regions and provinces in Afghanistan.\n",
            "Description: The dataset contains information on the number of civilian houses, vehicles, farms, local businesses, and markets in different regions of Yemen, including Sana'a, Sa'ada, Ibb, Aden, and Hudaydah. The data includes the respective quantities for each category in each region. For example, Sa'ada has 1311 civilian houses, 77 civilian vehicles, 377 farms, 13 local businesses, and 21 markets. Overall, the dataset provides a snapshot of the civilian infrastructure in these regions of Yemen.\n",
            "Description: The dataset contains information on casualties in Yemen, with columns for the date and the number of casualties. The data spans from December 2017 to August 2018, with casualty numbers ranging from 277 to 919. The dataset provides a snapshot of the casualties recorded each month during this period.\n",
            "Description: The dataset from the file \"Zimbabwe Baseline Population3.xlsx\" contains information related to the JIAF Severity Scale Reference Table. The data includes definitions for the severity scale and methodology for distributing the total number of people in an area or affected group across five severity classes. The dataset appears to provide a framework for categorizing and analyzing the severity of a situation based on specific criteria.\n",
            "Description: The dataset contains information on registered refugees and other registered persons in the State of Palestine, specifically in the Gaza Strip and the West Bank. The data includes the number of registered refugees and other registered persons in each region. In the Gaza Strip, there are 1,460,315 registered refugees and 161,806 other registered persons, while in the West Bank, there are 858,758 registered refugees and 207,014 other registered persons.\n",
            "Description: The dataset contains information on humanitarian response activities related to COVID-19, including details such as lead organization, implementing partners, cluster/sector, region, province, city/municipality, activity status, start and end dates, language, communication channels, and vulnerable groups. The data includes ongoing, completed, and planned activities conducted by organizations like Action Against Hunger in various regions of the Philippines, focusing on community engagement and information sharing/awareness raising. The dataset also includes codes for regions, provinces, and city/municipalities.\n",
            "Processed 61.0% resources ...\n",
            "Description: The dataset contains information on language demographics in Peru, with columns indicating the main language spoken, its share in the population, and the number of named languages. The data includes details on the population size, gender distribution, and literacy rates. The main language in Peru is Spanish, spoken by 84.3% of the population, followed by Quechua, Aymará, and Asháninka. The dataset also includes information on other indigenous languages spoken, as well as general notes on data confidence and metadata.\n",
            "Description: The dataset from the file \"bfa_hno_20213.xlsx\" contains two columns, \"Unnamed: 0\" and \"Unnamed: 1\", with the first row being headers. The data in the dataset includes a sum of \"Nombre total de PDI\" (Total number of PDI) which is 973956 and a sum of \"Total PIN\" which is 3533773.02737.\n",
            "Description: The dataset contains information on different townships in Myanmar, including the number of people in need and targeted for assistance in 2021. The data includes categories such as Internally Displaced Persons, IDP returnees, non-displaced stateless people, and other vulnerable crisis-affected individuals. The data also provides percentages for females, children, adults, elders, and disabled individuals within each category. The dataset is structured with rows representing different townships and columns representing specific data points for each category.\n",
            "Description: The dataset contains information on various organizations, including their full names, sectors, sub-sectors, activities, and locations. The data includes organizations such as Action Against Hunger and ACTED, with activities ranging from health-related interventions to protection monitoring. The locations mentioned in the dataset include Benghazi, Sebha, Derna, and Wadi Ashshati. The dataset appears to be related to humanitarian and development activities, with a focus on COVID-19 response and cash assistance programs.\n",
            "Description: The dataset from the file \"Burundi Cankuzo_FD3.xlsx\" contains information on various organizations operating in the Cankuzo region of Burundi. The data includes details such as the timestamp, data source, organization name, region, country, province, and sector of operation. Organizations mentioned in the dataset are involved in sectors like food security, protection, SGBV (Sexual and Gender-Based Violence), health, environment protection, and nutrition. The dataset seems to capture data entries from different dates, with each entry providing specific information about the organizations and their activities in the region.\n",
            "Description: The dataset contains information on air- and ground-launched explosive weapons affecting health facilities from 2016 to 2020, with details such as SHCC Event Code, Date, Country, Type of location, Explosive weapon type, and Perpetrator. The data includes events in countries like Syria and Yemen, involving both air-launched and ground-launched weapons, with perpetrators identified as State Forces or No Information.\n",
            "Description: The dataset contains COVID-19 data for Senegal at the subnational level, with columns including ID, DATE, ISO_3, PAYS, ID_PAYS, REGION, ID_REGION, CONTAMINES, DECES, GUERIS, CONTAMINES_FEMME, CONTAMINES_HOMME, CONTAMINES_GENRE_NON_SPECIFIE, NOUVEAUX_INDIVIDUS_VACCINES, and TOTAL_INDIVIDUS_VACCINES. The data includes information on the number of infected, deceased, recovered individuals, as well as gender-specific cases and vaccination details. The dataset is sourced from the Ministry of Health and provides information on different regions within Senegal.\n",
            "Description: The dataset provided is an Excel file containing criteria for selecting indicators related to humanitarian consequences. The criteria include relevance, accuracy, precision, uniqueness, coverage, sampling base, timeliness, and level of analysis. The document outlines specific considerations for each criterion, such as the indicator's clear relationship to the humanitarian consequence measured, its ability to measure what it intends to, and the importance of non-redundant indicators. Additionally, it emphasizes the need for data sources to be representative, comparable in sampling base, timely, and collected at a consistent level of analysis for easier aggregation.\n",
            "Description: The dataset contains information related to site assessments for various locations affected by natural disasters in Peru. It includes details such as site names, assessment dates, site statuses, GPS coordinates, number of households and individuals affected, access to various services, reasons for displacement, and presence of management committees or agencies at the sites. The data also covers aspects like security incident reporting, access to education, shelter, health, food, income, and referral mechanisms for protection concerns. Additionally, it captures information on different types of disasters, conflict or violence, and demographic details of the affected populations.\n",
            "Description: The dataset contains information on various aspects related to displacement camps in Kenya, including details on site locations, demographics, services provided, health and sanitation conditions, access to education, livelihood activities, and security measures. Each entry includes data such as site ID, location, survey date, accessibility, registration activities, management committees, service providers, support services provided (e.g., WASH, health, shelter, food), site classification, shelter types, ownership of land, origin of displaced groups, reasons for displacement, security measures, access to resources (e.g., food, water, electricity), health problems, access to health facilities, education services, income-generating activities, livestock presence, and intended return or relocation plans. The dataset also covers aspects like living conditions, access to basic amenities, waste disposal, food sources, malnutrition screening, health problems prevalent at the sites, access to health facilities, education services, income sources, and livestock presence.\n",
            "Processed 63.0% resources ...\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador_site_assessment_R7_Pb3.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 29776 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information related to surveys conducted in various locations, including details such as survey date, site ID, site name, location information, demographics of evacuees (including age and gender breakdown), shelter types, origin villages, evacuation details, health problems, access to services, and concerns of the affected population. The data also includes information on water sources, sanitation facilities, food distribution, health issues, education access, security availability, gender-based violence mechanisms, and topics of most concern. Each entry in the dataset represents a specific survey conducted in different areas, providing insights into the conditions and needs of the affected populations.\n",
            "Description: The dataset contains information about various villages and settlements, including demographic data such as population breakdown by age and gender, affected households, persons with special needs, movement intentions, disaster management, shelter data, water, sanitation, and hygiene data, food and nutrition data, health data, education data, safety and protection data, communication data, and more. The data is structured in a tabular format with multiple columns representing different variables for each village or settlement. The dataset includes details such as village names, district, province, location coordinates, household sizes, affected population, disaster impact, relief assistance needs, and community resources.\n",
            "Description: The dataset contains information on operational presence in Afghanistan from January to March 2020, with columns including region, province, district, organization acronym, organization name, organization type, cluster sector code, cluster sector name, and a question field. The data shows various organizations operating in different provinces and districts, with a focus on sectors such as health and nutrition. The dataset provides details on the presence of organizations like the Agency for Assistance and Development of Afghanistan (AADA) in different regions, provinces, and districts for specific sectors.\n",
            "Description: The dataset contains information on health care-related incidents in Nagorno-Karabakh in 2020, involving various factors such as event codes, incident dates, countries, perpetrators, weapons used, attack types, attack contexts, and impacts on health workers and facilities. The data includes details on health workers killed, kidnapped, arrested, threatened, injured, assaulted, and subjected to sexual violence, as well as attacks on facilities resulting in destruction or damage. It also covers incidents like armed entry into medical facilities, looting of health supplies, and damage to health transportation. The dataset provides a comprehensive overview of the health care-related security situation in Nagorno-Karabakh during the specified period.\n",
            "Description: The dataset contains information on the number of people in need (in thousands) in different provinces of the Democratic Republic of the Congo (DRC) in 2021, along with the percentage breakdown of women, men, children, adults, elderly persons, and people with disabilities within the population in need. The data includes provinces such as Bas-Uele, Equateur, Haut-Katanga, Haut-Lomami, Haut-Uele, Ituri, Kasaï, Kasaï-Central, and Kasaï-Oriental. The majority of provinces have a fairly equal distribution between men and women in need, with varying percentages of children, adults, and other demographic groups.\n",
            "Description: The dataset contains information on various counties in South Sudan for the year 2021, including details such as population in thousands, number of people in need in thousands, and levels of need categorized as minimal, stress, severe, extreme, and catastrophic. The data is organized by county, region, and specific indicators for each region. The dataset provides a comprehensive overview of the humanitarian needs in different areas of South Sudan, highlighting the population affected and the severity of their needs.\n",
            "Description: The dataset from the file CAF_HRP_20212.xlsx contains information on different prefectures in Central African Republic for the year 2021. It includes data on population, population in need, acute population in need, and target population for each prefecture. The dataset lists the prefecture names, year, cluster, total population, population in need, acute population in need, and target population. The data provides insights into the humanitarian needs and target populations in various regions of the country for the specified year.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various organizations, partners, clusters, sub-clusters, regions, provinces, cities/municipalities, barangays, evacuation sites, activities, and their statuses (ongoing, completed, planned). Each entry includes details such as the organization name, partner organization, sector, start and finish dates, and additional remarks. The activities range from construction projects to distribution of hygiene and sanitation materials. The data pertains to response efforts for Typhoon Goni (Rolly) and Vamco (Ulysses) in the Philippines, specifically in Region V (Bicol Region) with a focus on WASH and health-related initiatives.\n",
            "Description: The dataset contains information on various sectors such as Nutrition, ES NFI, WaSH, Education, Child Protection, FSL, Health, GBV, Protection, RCF, and Total for different regions including Abyei, Al Gezira, Blue Nile, Central Darfur, East Darfur, Gedaref, Kassala, and Khartoum. The data includes details like targets, reached values, and percentages for each sector in the year 2020. The dataset provides a comprehensive overview of the response and achievements in these sectors across different regions during the specified time period.\n",
            "Processed 65.0% resources ...\n",
            "Description: The dataset contains information related to Ebola cases in Guinea in 2021, with columns including date, ISO_3 country code, country name, confirmed and suspected Ebola cases, gender breakdown of cases, contact cases, hospitalizations, deaths, recoveries, and vaccinations. The data shows fluctuations in these metrics over time, with some missing values and negative values indicating inconsistencies in the data. The dataset appears to be structured with headers and rows of data, providing a snapshot of the Ebola situation in Guinea during the specified time period.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea_Ebola_Subnational_2021_HXL_HERA2.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea_Ebola_Subnational_2021_HXL_HERA2.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea_Ebola_Subnational_2021_HXL_HERA2.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea_Ebola_Subnational_2021_HXL_HERA2.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea_Ebola_Subnational_2021_HXL_HERA2.csv ... Error tokenizing data. C error: Expected 1 fields in line 3, saw 2\n",
            "\n",
            "Description: The dataset contains information on various countries, including their ISO codes, names, access indicators such as % of visas pending or denied and % of travel authorizations or movements denied, number of incidents reported in previous years, and % of CERF and CBPF projects affected by insecurity and inaccessibility. It also includes details on vaccine campaigns, their statuses, and the number of learners enrolled from pre-primary to tertiary education in each country. The data provides insights into access issues, security concerns, and educational enrollment across different nations.\n",
            "Description: The dataset contains weekly COVID-19 data for the Moscow region, with columns for the date, number of sick individuals, number of individuals who have recovered, and number of deaths. The data spans from January 27, 2021, to February 4, 2021. The number of sick individuals ranges from 919,662 to 937,450, the number of recoveries ranges from 791,320 to 835,168, and the number of deaths ranges from 13,175 to 13,701 during this period.\n",
            "Description: The dataset contains population projection data for different regions in the Central African Republic for the year 2021. It includes information such as the name and code of the prefecture and sub-prefecture, total population, population excluding refugees, and population distribution by age and gender categories (0-18, 19-59, 60+). The data provides a detailed breakdown of population demographics for each region, including the number of males and females in each age group.\n",
            "Description: The dataset contains information on language demographics in Mali, with columns representing administrative regions, language data, population demographics, literacy rates, and metadata details. The data includes the names of main languages spoken, their shares in the population, and literacy rates among males and females. The dataset also provides information on the confidence level of the data and includes notes. The dataset covers 15 named languages, with Bambara being the most prevalent language. The total population is approximately 14.5 million, with slightly more females than males. Literacy rates vary among different language groups, with a focus on French literacy rates for individuals aged 12 and above.\n",
            "Description: The dataset contains information on various organizations operating in Kayanza, Burundi, including CARE International, World Food Program, Action Aid, Fédération Nationale des Associations engagées dans le Domaine de l'Enfance au Burundi (FENADEB), Initiative Pastorale pour la Réinsertion des Enfants en Difficulté (IPRED), Observatoire INEZA des droits de l'enfant au Burundi (OIDEB), and Red Cross - Burundi. The organizations are involved in sectors such as education, food security, and child protection, with activities ranging from development to humanitarian aid. The dataset includes details like organization type, web links, and sector types.\n",
            "Description: The dataset contains information on Covid-19 cases in different provinces of DR Congo as of December 14, 2020. It includes data on the number of health structures, total affected individuals, confirmed cases, and probable cases in each province. The dataset also provides the data source for each entry. The provinces listed include Bas-Uélé, Equateur, Haut-Katanga, Haut-Lomami, Haut-Uélé, Ituri, Kasaï, Kasaï-Central, and Kasaï-Oriental.\n",
            "Description: The dataset contains information on various organizations, their implementation agencies, sectors, modalities, statuses, locations (province, district, sub-district, village), and dates related to aid activities. The data includes details such as dates, organization names, implementation agencies, sectors, modalities, statuses, and location information at different administrative levels. The dataset captures planned and in-progress activities related to logistics, displacement and protection, food security, and socio-economics, with varying start and end dates for each activity.\n",
            "Description: The dataset contains information about various organizations operating in Kirundo, Burundi, including their names, types, sectors, and activities. The data includes details such as timestamps, data sources, organization names, organization types, region, country, province, and sector types. Some organizations mentioned are involved in areas like food security, protection, and humanitarian efforts. The dataset also includes URLs for some organizations.\n",
            "Description: The dataset contains information on various organizations operating in Bujumbura Mairie, Burundi, as of April 2020. The data includes details such as the organization name, type (e.g., International NGO, National NGO), sector (e.g., Child Protection, Humanitarian), and the type of activity they are involved in. The dataset also includes timestamps indicating when the information was reported. The majority of organizations listed are National NGOs, with a focus on Child Protection and Humanitarian activities. Some entries have missing information, such as the sector or type of activity.\n",
            "Processed 67.0% resources ...\n",
            "Description: The dataset contains information on COVID-19 cases in Ethiopia, with columns including ID, Date, ISO_3, Country, Infected, Cumulative_Infected, Recovered, Cumulative_recovered, Deceased, Cumulative_deceased, and Active_cases. The data starts from January 1, 2020, with all counts initially at 0. The dataset appears to track daily changes in the number of infected, recovered, deceased, and active cases in Ethiopia due to COVID-19.\n",
            "Description: The dataset contains information from a baseline assessment in Armenia, with columns including Date Reported, Country, UniqueCode, Region, Marz, Community name, Settlement Name, Settlement code, number of individuals, number of families, average family size, number of males, and number of females. The data provides details on various communities within the Aragatsotn region of Armenia, including the number of affected individuals and families in each settlement, as well as the gender distribution. The assessment was conducted on December 11, 2020, and includes data on multiple settlements such as Agarak, Agarakavan, Aghdzk, Antarut, Ashtarak, Aparan, Avan, Aragats, and Aragatsavan.\n",
            "Description: The dataset contains information on food prices in Italy, with columns including date, item name, unit, category, price, currency, country, market name, and various identifiers. The data includes details such as the date, item name (e.g., Apples), unit (KG), category (vegetables and fruits), price (0.67 USD), and market name (National Average). The dataset seems to provide information on retail prices for different food items in Italy, with the first row serving as a header for the columns.\n",
            "Description: The dataset contains information on different sectors/interventions being implemented in various localities within states. It includes details such as the state, locality, sector/intervention, implementing organization name, and implementing organization type. The data shows a variety of sectors being implemented by different organizations, including coordination, education, and food security and livelihoods (FSL). The implementing organizations range from UN agencies like UNOCHA and UNICEF to international NGOs like BGRRF and local NGOs like GAH and ACAD.\n",
            "Description: The dataset contains information related to site assessments in Guatemala, specifically focusing on various aspects of shelters or sites providing assistance to individuals affected by disasters. The data includes details such as reported dates, location information, number of families and individuals at the site, presence of vulnerable populations, availability of basic services like water and sanitation, health conditions, infrastructure damages, and needs for non-food items. Additionally, it covers aspects like waste disposal mechanisms, access to healthcare, COVID-19 cases, food provisions, and support from different entities. The dataset provides a comprehensive overview of the conditions and requirements of these sites in Guatemala.\n",
            "Description: The dataset contains information related to site assessments and responses to hurricanes ETA and IOTA in Honduras. It includes data such as reported dates, interview dates, location details (country, department, municipality), site names, opening dates, shelter types, reporting to COEN, total families and individuals at the site, demographics (pregnant women, lactating mothers, disabled individuals), feedback mechanisms, security incidents, services availability (electricity, phone coverage), waste disposal methods, sanitation facilities, first aid availability, access to healthcare, COVID-19 cases, meal provisions, infrastructure damages, presence of foreign migrants or refugees, water sources, waste disposal methods, gender-separated facilities, food and non-food needs, and other specific details for each site.\n",
            "Description: The dataset stored in the file ni_lang_v01_admin02.csv contains information related to Nicaragua, including columns for the country's name, code, indigenous language share, number of indigenous language speakers, total population, data confidence level, and additional notes. The data shows that Nicaragua has an indigenous language share of 0.057, with 293,086 indigenous language speakers out of a total population of 5,154,855. The data confidence level is medium, and there are some missing values in the dataset.\n",
            "Description: The dataset contains information about different collective sites in Cox's Bazar, including details such as site names, location coordinates, total households, total individuals, family size average, shelter needs, safety concerns, water sources, bathing practices, latrine access problems, main food sources, income sources, income source changes, nutrition barriers, health access problems, education barriers, safety problems, incident reporting, interactions with the host community, feedback mechanisms, mobile phone ownership, severe needs, credibility of information, and consistency of key informant data. The data provides insights into the living conditions, challenges, and needs of the populations in these collective sites.\n",
            "Description: The dataset contains information on key figures related to humanitarian crises, including the date, country, number of people in affected areas, displaced people, people in need, severely acute malnourished (SAM) children, and food insecure people aged 3 and above. The data spans multiple dates and countries, with varying numbers of affected individuals in each category. The dataset provides a snapshot of the humanitarian situation in different countries over time, highlighting the scale of displacement, food insecurity, and other challenges faced by populations in crisis.\n",
            "Description: The dataset contains COVID-19 subnational data for Benin, with columns including date, ISO code, country name, region, number of confirmed cases, deaths, recoveries, gender-specific cases, data source, and link to the source. The data is organized by region and includes information such as the number of infected individuals, deaths, and recoveries reported on March 16, 2020. The source of the data is VOA Afrique, and the dataset provides a detailed breakdown of COVID-19 cases at the regional level in Benin on the specified date.\n",
            "Processed 68.0% resources ...\n",
            "Description: The dataset contains information on flood data by state and locality, including the number of households affected, number of people affected, number of deaths, infrastructure damage, education impact, health impact, agriculture impact, animal impact, latrine damage, water damage, and other impacts. The data includes details such as the state, locality/village, extent of destruction (completely destroyed or partially damaged), and specific numbers for each category. The dataset provides a comprehensive overview of the impact of floods in various regions.\n",
            "Description: The dataset contains information on COVID-19 cases in Gambia, with columns including ID, date, ISO_3 code, country name, region, number of confirmed cases, deaths, recoveries, and gender breakdown of cases. The data is sourced from the Ministry of Health and sanitation, with each entry representing a specific region within Gambia and its corresponding COVID-19 statistics. The dataset provides a detailed breakdown of cases by region and gender, allowing for analysis of the spread and impact of the virus in different parts of the country.\n",
            "Description: The dataset contains information on funding requirements and funding received for activities in the country with the country code ESH. The data includes columns for country code, activity ID, name, start and end dates, year, funding requirements in USD, total funding received in USD, and the percentage of funding received. The dataset spans from 2006 to 2018, with various funding amounts recorded for each year.\n",
            "Description: The dataset contains information related to a baseline assessment of floods in Nepal, with columns representing various attributes such as reported date, country, province, district, municipality, ward number, displaced population in the ward, total locations, population in different locations (host community, open space, village, community center, evacuation centers, and other), and population counts. The data includes details on affected individuals, location information, and population figures for different areas within Nepal.\n",
            "Description: The dataset contains survey information related to displacement conditions, site management, demographics of internally displaced persons (IDPs), vulnerable groups, shelter and non-food items (NFI), water, sanitation, hygiene (WASH) facilities, food access, health issues, education, livelihood, protection, information sources, and comments. Each entry includes details such as survey date, site location, IDP demographics, displacement conditions, shelter and NFI assistance received, water sources, toilet facilities, food access, health problems, education access, livelihood status, protection availability, information sources, and any additional comments. The data provides insights into the needs and conditions of displaced populations in Sri Lanka following a disaster in May 2017.\n",
            "Description: The dataset contains information related to site assessments in Bolivia, including details such as site names, classifications, geographical coordinates, site status, support services provided (e.g., water, sanitation, health, education), demographics of displaced populations, living conditions (e.g., shelter types, access to basic amenities), health and nutrition status, access to education and income-generating activities, security measures, community relations, and information dissemination methods. Each entry includes specific data points such as dates of reporting and occurrence, site IDs, location details, agency involvement, and various indicators related to the living conditions and needs of the displaced populations.\n",
            "Description: The dataset contains information on site assessments in Antigua and Barbuda, including details such as site names, assessment dates, site statuses, opening and closing dates, reasons for displacement, site types, GPS coordinates, administrative codes, number of households and individuals, place of origin of IDP groups, presence of site management committees and agencies, security incident reporting, access to various services, reasons for displacement, and round numbers. The data also includes information on security incidents, access to water, sanitation, hygiene, education, shelter, non-food items, health, food, income, and referral mechanisms for protection concerns. The dataset covers multiple sites with varying characteristics and responses to questions related to site management and services.\n",
            "Description: The dataset contains information on access constraints and their impact due to the COVID-19 outbreak in various countries. It includes columns such as country code, country name, percentage of identified access constraints, and specific access constraints related to movements of goods and personnel. Additionally, it covers impacts like restrictions on humanitarian operations, violence against personnel, and increased costs. Each row represents a different country, detailing the extent of access constraints and their impact.\n",
            "Description: The dataset contains information on various administrative regions in Zimbabwe, including the mean number of days households in each region use a specific coping strategy per week. The data includes details such as the survey date, administrative region names, administrative strata, and demographic information. The dataset is structured with columns for variables like country name, indicator code, indicator name, and indicator value. The dataset provides insights into household coping strategies across different regions in Zimbabwe.\n",
            "Description: The dataset provided is in an Excel file format and contains information related to various regions and indicators in Mali. The data includes columns such as regions, codes, and different indicators like education, nutrition, and health. Each row represents data for a specific region, with values for different indicators. The dataset seems to focus on metrics related to returnees, shelters, education, nutrition, protection, health, and other sectors in different regions of Mali.\n",
            "Processed 70.0% resources ...\n",
            "Description: The dataset contains information on various organizations involved in COVID-19 related activities in Papua New Guinea. It includes details such as organization name, acronym, type, sector/cluster, activity, implementation modality, status, location (province, district, municipality), number of beneficiaries, and type of beneficiaries. The organizations mentioned are a mix of international NGOs, national NGOs, and private sector entities. The activities range from remote psychological support and referrals to safe houses, counseling, and capacity-building initiatives. The dataset also highlights efforts to integrate COVID-19 awareness and prevention measures into existing projects, particularly focusing on gender-based violence, child protection, and youth empowerment.\n",
            "Description: The dataset contains information on Covid-19 cases in Mali, with columns for Date, Région (Region), Cercle (Circle), Sum of Confirmed Cases, Sum of Deaths, Sum of Recovered Cases, Sum of Suspected Cases, and Sum of Contact Numbers. The data includes details such as the number of confirmed cases, deaths, and recoveries in different regions and circles on specific dates. The dataset provides a snapshot of the Covid-19 situation in Mali over time, showing the progression of cases in different areas.\n",
            "Description: The dataset contains information on the Ebola epidemic national figures, with columns including publication date, report date, country, confirmed cases, probable cases, total cases, confirmed deaths, total deaths, total suspected cases, new deaths, new cured, total cured, new suspected cases, old suspected cases, and various change values. The data pertains to the Democratic Republic of the Congo (COD) and includes details such as the number of confirmed, probable, and total cases, as well as deaths and recoveries. The dataset also includes sources for the information provided.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/time_series_covid19_confirmed_global_iso3_regions2.csv ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 71013 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on various sites in El Salvador used as shelters, including details such as location, site status, organization managing the site, demographics of individuals at the site, facilities and services available (such as water supply, sanitation, security, and healthcare), needs for non-food items, presence of support organizations, and infrastructure damages. The data also includes geolocation coordinates for each site.\n",
            "Description: The dataset contains information on COVID-19 cases in Guinea at a subnational level, with columns including date, ISO code, country name, region, number of confirmed cases, deaths, recoveries, and gender breakdown. The data is sourced from the ANSS Guinea social media platforms. The dataset provides details on different regions within Guinea and their respective COVID-19 statistics as of March 12, 2020.\n",
            "Description: The dataset contains information on COVID-19 cases in different regions of Liberia, with columns for the date, ISO code, country name, region name, number of confirmed cases, deaths, recoveries, gender distribution, and data source. The data is structured in a CSV format with semicolon delimiters. The dataset provides details on the affected individuals in various regions of Liberia as reported by the National Public Health Institute of Liberia on March 16, 2020.\n",
            "Description: The dataset contains information on the prevalence of various diseases in outpatient clinics per state in Sudan. The data includes the number of cases for diseases such as Acute Tonsillitis, Diabetes Mellitus, Diarrhoea Gastroenteritis, Diseases Of Respiratory System, Dysentery, Essential Hypertension, Malaria, Other Disorders Of Urinary Tract, Pneumonia, and Typhoid in different states. Each row represents a state, and the corresponding number of cases for each disease is provided. The dataset can be used to analyze and compare the prevalence of these diseases across different regions in Sudan.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/co_lang_v012.xlsx contains information on the number of named languages, main languages, language shares, population demographics, literacy rates, and metadata confidence for Colombia. The data includes details such as the main language being Spanish with a share of 99.52%, along with other languages spoken in the country. The total population is approximately 40.57 million, with a slightly higher female population. The literacy rate for individuals aged 3 and above is around 84.7%, with similar rates for both males and females. The dataset also indicates the presence of language data, confidence levels, and additional notes.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/mhu-jan-dec2019-idn-hxl2.csv ... Error tokenizing data. C error: Expected 1 fields in line 4640, saw 2\n",
            "\n",
            "Description: The dataset stored in the file \"kh_lang_v012.xlsx\" contains information on various languages spoken in Cambodia, including the number of named languages, the main language spoken, and its share among the population. The data includes details on languages such as Khmer, Vietnamese, Chinese, Laotian, Jarai, Western Cham, Kuy, Krung, Phong-Kniang, Lave, Tampuam, Kraol, and others. Additionally, demographic information such as total population, male and female population, literacy rates, and metadata details are provided. The dataset indicates that Khmer is the main language spoken by 93% of the population in Cambodia.\n",
            "Processed 72.0% resources ...\n",
            "Description: The dataset contains information related to various sectors such as Health, FSL, WaSH, Nutrition, RRR, General Protection, GBV, Education, Child Protection, ES NFI, RCF, Mine Action, and others. It includes data on different targets, including IDPs, Returnees, Vulnerable Residents, Refugees, and Total Targets. The data is organized in columns with information on priority, PIN, IDPs, Returnees, and other relevant details for different localities within different sectors. The dataset also includes information on different areas within Al Gezira, such as El Hasaheesa, El Kamleen, El Manageel, and others, with corresponding values for targets and other related metrics.\n",
            "Description: The dataset contains information on various countries' Humanitarian Response Plans for 2020, including the number of people in need, internally displaced persons (IDPs), refugees, and returnees. The data includes countries like Afghanistan, Burkina Faso, Burundi, Cameroon, Central African Republic, Chad, Colombia, Democratic Republic of Congo, and Ethiopia. The dataset provides figures on the population in need, the number of IDPs, refugees, and returnees for each country.\n",
            "Description: The dataset contains COVID-19 projections for various countries, including total cases and total deaths. The data includes information such as the country name, country ISO code, minimum and maximum total cases, as well as minimum and maximum total deaths. The dataset covers countries like Afghanistan, Burkina Faso, Burundi, Cameroon, Central African Republic, Chad, Colombia, Democratic Republic of the Congo, and Ethiopia. The projections provide estimates of the affected population in terms of cases and deaths for each country.\n",
            "Description: The dataset contains information on people reached by state in Sudan from January to December 2018 across different sectors. The data includes columns for State, Sector, and the number of people reached. The dataset also includes some unnamed columns and a column labeled \"#reached+sum\" which appears to show the total number of people reached. The data shows varying numbers of people reached in different states and sectors, with some missing values for certain entries.\n",
            "Description: The dataset contains information on people reached by locality in Sudan from January to December 2019, with columns representing different sectors such as Food Security and Livelihoods (FSL), Nutrition, Child Protection, Health, Water, Sanitation, and Hygiene (WaSH), Education, Refugee Assistance (RCF), Emergency Shelter and Non-Food Items (ES NFI), and Gender-Based Violence (GBV). Each row corresponds to a specific locality within a state in Sudan, detailing the number of people reached in each sector. The dataset includes data on various localities within different states, providing insights into the humanitarian assistance and services provided in these areas during the specified time period.\n",
            "Description: The dataset contains information on COVID-19 cases and deaths in different states, with columns for date, state, FIPS code, number of cases, and number of deaths. The data includes entries for various states such as Alabama, Alaska, Arizona, and California, among others, with corresponding figures for affected/infected individuals and those who have died. The dataset appears to be structured with headers for each column and data entries below, providing a snapshot of the COVID-19 situation in different states as of the specified date.\n",
            "Description: The dataset stored in the file th_lang_v012.xlsx contains information on language demographics in Thailand. It includes columns such as admin0_name, admin0_pcode, number_of_named_languages, main_language, main_language_share, and language-specific data percentages. The data also includes population totals, gender breakdowns, literacy rates, and metadata details. The dataset specifically focuses on the prevalence and distribution of different languages in Thailand, with Thai being the dominant language. The dataset provides insights into population demographics, literacy levels, and language diversity in the country.\n",
            "Description: The dataset contains information on global coordination groups, including HRinfo ID, preferred terms in English and French, acronyms, group types, homepages, creation and update dates, and notes. The data includes details on various clusters such as Emergency Shelter and NFI, Camp Coordination/Management, Mine Action, Food Security, Water Sanitation Hygiene, Logistics, Child Protection, Protection, and Education. The dataset was created on 03/16/2017 and last updated on 4/4/2018, with some entries having added French names and updated information based on feedback.\n",
            "Description: The dataset contains information on various indicators related to fertility rates and contraceptive use among married women in different regions of Turkey in 1993. The data includes indicators such as total fertility rate and the percentage of married women currently using any method of contraception. Each entry specifies the region, indicator value, precision, country code, survey year, survey ID, and other relevant details. The dataset also includes metadata columns such as ISO3, Location, DataId, IndicatorId, and CharacteristicId.\n",
            "Description: The dataset contains information related to various indicators for Turkey in 1993, sourced from the DHS Quickstats Data. It includes data such as total fertility rate, contraceptive use among married women, unmet need for family planning, infant mortality rate, median age at first marriage for women, and under-five mortality rate. Each entry specifies the indicator, its value, precision, country code, survey details, and characteristic information. The dataset also includes indicator IDs, order, types, and labels, along with denominator weights and confidence intervals.\n",
            "Processed 74.0% resources ...\n",
            "Description: The dataset contains information on the distribution of population by sex, number of households, land area, and population density across different counties and sub-counties. The data includes total population, male and female populations, land area in square kilometers, and population density. The dataset also includes various unnamed columns with additional information that may need further clarification or cleaning. The dataset covers multiple counties and sub-counties, providing a detailed breakdown of demographic and geographical data for analysis and comparison.\n",
            "Description: The dataset contains information on sectors such as Food Security and Agriculture, Nutrition, Health, and WASH in DPRK, with data on total people in need, total people targeted, percentage of people targeted, male and female breakdowns, and breakdowns by age groups. The data shows the number of people in need, targeted, and the percentage targeted for each sector. Additionally, there is a total figure that accounts for double counting.\n",
            "Description: The dataset contains information on disease outbreaks in different states of Sudan, including the cumulative number of cases for diseases such as Dengue, Chikungunya, Rift Valley Fever, Diphtheria, and Cholera. The data shows varying numbers of cases across different states, with Kassala having the highest number of Dengue cases (3961) and Cholera cases (204). Other states like Red Sea and River Nile also have significant numbers of cases for certain diseases. Overall, the dataset provides a snapshot of the disease burden in Sudan during the specified time period.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Conditional Formatting extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various programs and projects related to refugee and other persons of concern in Eritrea. It includes details such as agency names, program codes, titles, outcomes, sectors, funding requirements and received amounts for different quarters, as well as additional information and comments. The programs cover a range of sectors including health, water sanitation, education, environment, capacity development, food security, gender empowerment, youth empowerment, and social protection. Each entry also specifies the pillar it falls under, the government of Eritrea (GoSE) representative, and the category of the program.\n",
            "Description: The dataset from the file \"200304_3W on NCDDS_EQ2.xlsx\" contains information on the status of projects, with a total of 1653 projects. Of these, 1241 projects are completed, 281 are ongoing, and 131 are planned. The data is structured in two columns, with the first column containing labels such as \"Completed,\" \"Ongoing,\" and \"Planned,\" and the second column showing the corresponding counts for each status category.\n",
            "Description: The dataset contains information on the Marawi conflict, with columns for administrative region names and the count of conflict statuses (Ongoing, Completed, Planned). The data includes regions such as Autonomous Region in Muslim Mindanao (ARMM), Basilan, Lanao del Sur, Maguindanao, Sulu, and Tawi-Tawi, with corresponding counts of conflict statuses. The dataset is stored in an Excel file located at the specified path.\n",
            "Description: The dataset located at /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/200210_Taal Eruption 3W_Consolidated2.xlsx appears to be empty, with no columns or index information available.\n",
            "Description: The dataset contains information on language demographics in Malawi, with columns representing administrative regions, language names, language shares, population demographics, literacy rates, and data confidence levels. The main language spoken in each region is listed along with its percentage share, as well as the percentage of the population that speaks English. Additionally, the dataset includes information on illiteracy rates among males and females, total population numbers, and data confidence levels. The dataset covers 13 named languages in Malawi and provides insights into language distribution and literacy levels across different regions.\n",
            "Description: The dataset contains location data for various UN agencies, including latitude and longitude coordinates, agency names, office types, clusters, and administrative information such as country, region, and administrative codes. The agencies represented include IOM, FAO, UNICEF, UNOCHA, WFP, and WHO, with offices located in countries like Benin, Burkina Faso, and Cameroon. The dataset provides a snapshot of the distribution and focus areas of these UN agencies across different regions.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/afghanistan-3w-operational-presence-october-to-december-20192.csv ... 'utf-8' codec can't decode bytes in position 15-16: invalid continuation byte\n",
            "Description: The dataset contains flow monitoring data for Yemen from January 2019 to December 2019. It includes information such as FMP code, reporting month and year, interview date, transportation means, departure and destination locations, demographics of migrants (total individuals, men, women, boys, girls), and flow type. The data also includes details on different nationalities of migrants and returnees, as well as the purpose of migration (economic migration for over 6 months). The dataset seems to be structured with columns representing various attributes related to flow monitoring and migration patterns.\n",
            "Processed 76.0% resources ...\n",
            "Description: The dataset contains information on site assessments for earthquake-affected areas in Mindanao, Philippines. It includes details such as site ID, survey round, location information, number of families and persons affected, displacement details, urgent needs, shelter/NFI, food security/livelihood, WASH, health, safety/security, and other observations. The data also highlights the designation of sites, property types, ownership, access types, management, and details on IDPs' origins and relocation prospects. Additionally, it mentions specific cases of reported health issues among IDPs and provides insights into the assistance received by farmers and laborers post-earthquake.\n",
            "Description: The dataset contains information on various humanitarian response plans, including details such as code, internal ID, start and end dates, plan version, categories, locations, years, original requirements, and revised requirements. The data includes flash appeals for natural disasters like droughts and hurricanes, as well as other response plans for events such as cyclones and floods. The dataset covers response plans for countries like Malawi, Grenada, Saint Vincent and the Grenadines, Bangladesh, Zambia, Zimbabwe, Burundi, Venezuela, and Niger, with corresponding financial requirements for each plan.\n",
            "Description: The dataset contains information related to site assessments in Bangui, specifically in Arrondissement 2, regarding the impact of floods in October 2019. The data includes details such as evaluation dates, geographical coordinates, number of displaced households and individuals, reasons for displacement, living conditions, security risks, access to water sources, health issues, access to medical services, education status of displaced children, food sources, market access, security concerns, and community relations. The assessments highlight various challenges faced by the displaced populations, including issues related to shelter, water access, sanitation, security, education, and healthcare. The dataset also captures the need for humanitarian assistance and the desire for a safe return to their original homes.\n",
            "Description: The dataset contains water quality data with columns including HH_ID, Filter_ID, Filter_type, #_sample, Date_sampling, Water_sample, EC_CFU/100ml, TC_CFU/100ml, Conductivity (µS/cm), Turbidity(NTU), ATP_by_lumitester_test(RLU), Monitoring_Period, State, Governorate, City, and City_code. The data includes information on different water samples such as raw water, filtered water, and cup water, along with corresponding values for E. coli and total coliform CFU/100ml, conductivity, turbidity, and ATP levels. The dataset pertains to water quality monitoring in the State of Palestine, specifically in Hebron and Jinba.\n",
            "Description: The dataset contains information on various sites in the Bahamas, including details such as site names, coordinates, administrative divisions, population statistics, shelter types, critical needs, water sources, garbage disposal options, food access, health problems, education facilities, available skills, livelihood opportunities, social assistance benefits, community relations, safety perceptions, security incidents, access to services, sources of information, community intentions, and more. It provides insights into the living conditions, challenges, and resources available in different areas, highlighting aspects related to shelter, water, food, health, education, livelihoods, social assistance, community dynamics, and information access.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia OCHA - Monitoring Matrix 20172.google sheet\n",
            "Description: The dataset contains information related to assessments conducted in various villages in South Sudan, including details such as location, round number, source round, date, administrative codes, accessibility, presence of internally displaced persons (IDPs) and returnees, demographic data on IDPs and returnees, reasons for displacement, shelter conditions, NFIs needed, water sources and purification methods, sanitation facilities, food access, health concerns, education access, security provision, incidents, and areas females avoid. The data also includes details on vulnerable populations, previous displacement, arrival dates, reasons for displacement, livelihood conditions, and hygiene promotion.\n",
            "Description: The dataset contains information on various sites in Mozambique, including details such as survey dates, site classifications, site names, location coordinates, site statuses, administrative divisions, site types, open dates, management agencies, site committees, accessibility, IDP origin regions, demographics, urgent needs, shelter conditions, NFIs, latrines, bathing spaces, hygiene practices, water sources, markets, food distribution, healthcare services, health facilities, access to education, security mechanisms, GBV survivor support, and psychosocial services. The data also covers aspects like water availability, market access, food distribution, healthcare access, disease reporting, education access, security measures, and support services for vulnerable populations.\n",
            "Description: The dataset contains information related to assessments and return logs in Logone Oriental, Chad. It includes data on various aspects such as population displacement, types of assistance provided, access to services like healthcare and education, water sources, sanitation facilities, security, food security, means of livelihood, communication networks, and general comments. The data is structured with columns representing different variables such as location, population demographics, assistance provided, health services, security, education, water sources, and market access. Each row represents a specific assessment or return log entry, detailing the conditions and characteristics of the site or village being assessed.\n",
            "Description: The dataset contains information related to assessments in different regions, including details such as evaluation dates, regions, departments, villages, GPS coordinates, population demographics, displacement reasons, transportation methods, assistance provided, water sources, sanitation facilities, health services, food security, communication networks, education access, and security concerns. The data includes responses from various sites and villages, detailing the conditions, needs, and challenges faced by displaced populations, returnees, and host communities. The dataset also captures information on the availability of services, infrastructure, and resources in these areas, providing insights into the living conditions and vulnerabilities of the populations surveyed.\n",
            "Description: The dataset contains information from assessments conducted in Mozambique after Cyclones Idai and Kenneth. It includes data on the impact of the cyclones on various aspects such as population displacement, shelter damage, livelihoods, access to basic services like water, health facilities, and schools, as well as agriculture and markets. The data includes details on the number of households and individuals affected, whether they received shelter kits, repaired their homes, and their main livelihood sources. It also covers access to farming lands, markets, health facilities, and schools, as well as the availability of water sources. The dataset provides insights into the challenges faced by the affected population and their recovery efforts following the cyclones.\n",
            "Processed 77.0% resources ...\n",
            "Description: The dataset contains information on various functional roles within humanitarian organizations, including details such as ID, preferred term, scope, usage, related terms, and translations in French and Spanish. The roles range from administrative officers and cluster coordinators to civil-military officers and public information officers. Each entry includes a description of the role's responsibilities and may also include notes on the role. The dataset was last updated on February 24, 2017, and includes a variety of functional roles and their corresponding details.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1129_GeoDist_HXL2.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1129_GeoDist_HXL2.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1129_GeoDist_HXL2.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1132_GeoDist2.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1132_GeoDist2.xls\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DI_Stat1132_GeoDist2.xls\n",
            "Description: The dataset contains information on various projects in Ethiopia, including details such as project status, start and end dates, donors, program organizations, implementing agencies, regions, zones, woredas, sector, and cluster activities. The data includes completed projects by organizations like VSO and USAID, with activities ranging from providing psychosocial training for teachers to purchasing learning stationery. The dataset also includes information on the type of organizations involved, such as international NGOs and international organizations.\n",
            "Description: The dataset contains information on natural disaster incidents in Afghanistan from 1 January to 31 December 2016. The data includes various types of natural disasters such as avalanches, earthquakes, flash floods, floods, heavy rainfall/snowfall, landslides, and mudflows. The incidents have been aggregated by district, with each record representing a unique event based on the type of natural disaster and date. The data is sourced from OCHA Field Offices and IOM, with additional information from HADB and OCHA reports. The dataset excludes droughts, fires, river bank erosion, and other disasters.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various organizations, partners, clusters, sub-clusters, regions, provinces, cities/municipalities, barangays, evacuation sites, activities, statuses, start and end dates, and remarks related to distribution of food packs for different families in different regions of the Philippines. The data includes details such as the number of families served, contents of the food packs, and additional items included. Each entry specifies the organization involved, the specific location, the type of activity conducted, and whether it was ongoing, completed, or planned. The dataset also includes region codes, province codes, and municipal city codes for reference.\n",
            "Description: The dataset contains information on various organizations, their types, partners, clusters, sub-clusters, regions, provinces, cities/municipalities, barangays, evacuation centers, activities, statuses, start and end dates, disasters/emergencies, and remarks. The data includes details such as the organization name, type, partner information, location, and activities conducted in the Autonomous Region in Muslim Mindanao (ARMM) region of Mindanao, excluding Marawi. Activities mentioned include assessments, distribution of relief items like NFI and hygiene kits, and post-distribution monitoring. The dataset seems to focus on humanitarian efforts and responses to human-induced disasters in the specified region.\n",
            "Description: The dataset contains information on internally displaced persons (IDPs) from the Marawi Conflict, with details such as province, city/municipality, codes, number of IDPs outside camps, inside camps, and total IDPs. The data includes provinces like Misamis Oriental and Lanao del Norte, along with specific cities and municipalities within these provinces. The total number of IDPs is provided for each location, both inside and outside camps. The dataset offers a comprehensive overview of the distribution of IDPs in different areas affected by the conflict.\n",
            "Description: The dataset contains information on the number of families displaced in different regions over reporting periods from June to August 2017 due to the Marawi conflict in the Philippines. It includes data on the number of evacuation centers, families inside the centers, families home-based, number of returned individuals, and the total displaced population. The regions covered in the dataset are ARMM, Others, and Region X. The data shows fluctuations in the displacement numbers over the specified time frame, with varying levels of displacement across the different regions.\n",
            "Description: The dataset contains information on natural disaster incidents in Afghanistan from 1 January to 31 December 2015. The data includes details on various types of natural disasters such as avalanches, earthquakes, flash floods, floods, heavy rainfall/snowfall, landslides, and mudflows. The incidents have been aggregated by district, and each record represents a unique event based on the type of natural disaster and date. The data is sourced from OCHA Field Offices and IOM, with additional information from HADB and other organizations. The dataset excludes droughts, fires, river bank erosion, and other disasters.\n",
            "Description: The dataset contains information on the number of people targeted for assistance each year from 2008 to 2016. The data shows an increasing trend in the number of people targeted for assistance over the years, starting at 28,000,000 in 2008 and reaching 97,600,000 in 2016.\n",
            "Description: The dataset contains registration details for different localities in Sudan, including information on the number of individuals in households categorized by age and gender, as well as specific characteristics such as being chronically ill, mentally ill, physically disabled, pregnant, single parents, lactating, and their employment status. The data also includes details on different types of occupations such as construction, agriculture, carpentry, cooking/bakery, nomads/shepherds, basic accounting, army/military, electricians, transportation, education, watchmen, nurses, midwives, business/commercial activities, metal smiths, barbers, wood collection, and others. The dataset provides a comprehensive overview of the demographics and socio-economic characteristics of the registered population in various regions of Sudan.\n",
            "Processed 79.0% resources ...\n",
            "Description: The dataset contains information related to various villages in the Lomami province of the Democratic Republic of Congo. It includes details such as village codes, dates of surveys, evaluation phases, village demographics, displacement and returnee statistics, reasons for displacement, and various indicators related to household needs and services like healthcare, education, water access, and security. The data also covers information on conflicts, food crises, and community interactions. Each entry provides specific details about the village, its evaluation status, and the challenges or assistance needed in the area.\n",
            "Description: The dataset contains information on humanitarian activities related to child protection in Mozambique, with details on organizations involved, types of activities, locations, beneficiaries targeted, and status of each activity. The data includes columns for leading and implementing organizations, donors, clusters, response domains, activities, indicators, outputs, modalities, and geographical information such as provinces, districts, and postos. The activities range from establishing child-friendly spaces to capacity strengthening and group discussions, targeting various beneficiaries including children, adults, boys, girls, men, women, and children with disabilities. The dataset also includes information on the start and end dates of each activity.\n",
            "Description: The dataset contains information related to humanitarian activities in Cabo Delgado, Mozambique. It includes details such as activity IDs, lead organizations, implementing partners, sectors/clusters, response activity types, descriptions, materials delivered, delivery modalities, locations (province, district, village), number of beneficiaries assisted, beneficiary types, start and end dates, and status of the activities. The data covers various organizations like ICRC, Ministerio Arco-Iris, OIM, and Caritas Pemba, and their efforts in providing relief through the distribution of non-food items to the general population.\n",
            "Description: The dataset contains information on site assessments in Burkina Faso, including details such as region, province, commune, type of settlement, displacement statistics, management organizations, population demographics, security issues, water sources, sanitation facilities, health services, food sources, education access, assistance received, communication networks, and community concerns. The data includes dates, geographical coordinates, and various indicators related to the living conditions and vulnerabilities of internally displaced persons in different regions and sectors.\n",
            "Description: The dataset contains information related to various aspects of displacement and humanitarian assistance in Papua New Guinea, specifically in the Ulawun Valcona region. The data includes details such as survey dates, site names, geographical locations, site management information, demographics of displaced populations, living conditions, health issues, education access, security measures, and community needs. Each entry provides specific data points such as the presence of site management committees, availability of services, health problems prevalent at the site, and intended return locations for displaced individuals. The dataset also includes information on the main topics communities are seeking information on, relocation opportunities, security incidents, and access to education services.\n",
            "Description: The dataset contains information on site assessments in Malawi, including details such as survey date, site ID, site name, site status, location data, number of IDP families and individuals, demographics, needs assessment, intentions to return home, reasons for return, assistance received, and barriers to return. The data includes various sectors of assistance provided by humanitarian partners, urgent unfulfilled needs, and factors influencing the decision to return home. Additionally, it captures information on the largest IDP group in each site, their origins, and their current situation.\n",
            "Description: The dataset contains information on sanitation access and open defecation in different areas within the State of Palestine, specifically in the Hebron Governorate. The data includes details such as the population in households, the number of households sharing toilets, the number of households without toilets, the sector (WASH), and the geographical coordinates (latitude and longitude) for each sub-location. The dataset provides a snapshot of sanitation conditions in various sub-locations within Hebron, highlighting the distribution of households with access to toilets and those without.\n",
            "Description: The dataset contains information on hard-to-reach districts in Yemen, with columns including administrative codes and names, number of people in need and in acute need, inter-cluster severity score, access status, and principal impediments. The data shows the breakdown of people in need and in acute need for various districts, along with their access status and severity score. The dataset provides insights into the humanitarian situation in these districts, highlighting areas with high levels of need and potential challenges in accessing aid.\n",
            "Description: The dataset contains information on disappearances in various municipalities, including the municipality code, department, municipality name, and the total number of disappearances for the years 2017, 2018, and 2019. The data shows the number of affected individuals in each municipality for the respective years.\n",
            "Description: The dataset contains information related to education facilities in the Cox's Bazar district, specifically in the Teknaf area. It includes details such as the name of the facility, facility ID, type of structure, status, geographical coordinates (latitude, longitude, altitude), and partners involved in implementing and supporting the programs. The facilities are categorized as learning centers or temporary learning centers, with each entry specifying the completion status. The dataset provides a comprehensive overview of various educational facilities in the specified region.\n",
            "Processed 81.0% resources ...\n",
            "Description: The dataset contains information on operational presence in Afghanistan from January to March 2019, with columns including region, province, district, organization acronym and name, organization type, cluster sector code and name, and a question field. The data shows multiple entries for different regions, provinces, and districts, all associated with the organization \"Save the Children Federation International\" operating in the emergency shelter and non-food items sector.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Afghanistan IDPs Returnees Needs Assessment May-Jun-182.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 22587 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on civilian casualties in different provinces of Afghanistan in 2018, with details on leading tactics or causes of casualties, total civilian casualties, deaths, injuries, and a comparison to 2017. The leading tactics or causes of casualties include suicide/complex attacks, ground engagements, IEDs, aerial attacks, targeted killings, and UXO/landmines. The provinces with the highest total civilian casualties are Kabul and Nangarhar, with various tactics contributing to the casualties. The dataset provides a snapshot of the impact of conflict on civilians in different regions of Afghanistan during the specified period.\n",
            "Description: The dataset edu5w01172.xlsx contains information on the number of targeted and reached people in different governorates. The data includes columns for Governorate, TargetedPeople, and ReachedPeople. Some entries have missing values for the number of reached people. The dataset includes information for governorates such as Anbar, Babylon, Baghdad, Basrah, Dahuk, Diyala, Erbil, Kerbala, and Kirkuk.\n",
            "Description: The dataset contains information on beneficiaries reached through quality education programs in hard-to-reach and besieged locations from January 2017 to September 2017. The data includes the number of children, youth, teachers, and education personnel benefiting from these programs, as well as the percentage of the targeted total reached. The dataset shows a steady increase in the number of beneficiaries reached each month, with some missing data for July 2017.\n",
            "Description: The dataset contains information on various aspects related to displacement camps in Ethiopia, including survey dates, site details, displacement reasons, site management committees, service providers, water sources, sanitation facilities, food distribution, health services, education access, livelihood activities, security, and information sources. Each entry includes details such as survey date, site ID, region, GPS coordinates, site type, accessibility, ownership, service providers, water sources, sanitation facilities, food distribution, health services, education access, livelihood activities, security incidents, and information sources. The data provides insights into the living conditions, needs, and preferences of displaced populations in Ethiopia.\n",
            "Description: The dataset contains information on displacement events in various countries, with columns including ISO3 country code, country name, start and end dates, year, conflict-related stock displacement, conflict-related new displacements, and disaster-related new displacements. The data shows the number of internally displaced persons (IDPs) due to conflict and disasters for different years in each country. For example, in Abyei Area, there were varying numbers of conflict-related IDPs in different years, while in Afghanistan, there were both conflict-related and disaster-related IDPs recorded over multiple years.\n",
            "Description: The dataset from the file bfa_pin_20192.xlsx contains information on regions, provinces, and the number of people in need and targeted for assistance in Burkina Faso. The data includes columns for country name, region, province, persons in need, persons targeted, admin 1 and admin 2 codes. Each row represents a different province within a region, with corresponding data on the number of people in need and targeted for assistance.\n",
            "Description: The dataset contains information on various programs and projects implemented by an international NGO across different regions and zones in Ethiopia. The data includes details such as organization type, project status, implementing agency, and activities related to sectors like agriculture, nutrition, and water, sanitation, and hygiene (WASH). The dataset also includes specific information on the locations (region, zone, woreda, and Pcode) where the projects are being carried out. The NGO, AAH, is shown to be actively involved in ongoing projects related to nutrition, WASH, and agriculture in multiple locations within the Amhara region.\n",
            "Description: The dataset contains information on various organizations, their types, regions, zones, woredas, sector projects, project statuses, and implementing partners. The data includes details such as organization names, organization types (national or international NGOs), regions, zones, woredas, woreda codes, sectors (e.g., WASH), project statuses (e.g., completed), and implementing partners. The dataset provides a snapshot of organizations involved in projects across different regions and sectors, along with their project statuses and implementing partners.\n",
            "Processed 83.0% resources ...\n",
            "Description: The dataset contains information related to Hurricane Michael Twitter data analysis, with columns representing date, hour, number of tweets, tweets related to affected individuals, infrastructure and utilities damage, injured or dead people, missing and found people, caution and advice, donation and volunteering, sympathy and support, and other useful reports. The data includes details such as the date and time of data collection, the count of tweets, and specific categories of tweets. The dataset appears to capture various aspects of the impact of Hurricane Michael on social media, including discussions on damage, support, and advice shared on Twitter during the specified time period.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Dashboard-reowa-sahel-20182.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Dashboard-reowa-sahel-20182.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Dashboard-reowa-sahel-20182.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Description: The dataset contains information on aerial surveys conducted during the Mozambique Cyclone Idai event. It includes columns such as uniqueid, deviceid, date, starttime, endtime, latitude, longitude, altitude, precision, grid, photo, severity, estimated population, and point of interest. The data provides details on various locations, severity levels of damage, estimated population affected, and types of structures impacted, such as houses, buildings, and airports.\n",
            "Description: The dataset contains information on displacement and return status of households and individuals in different provinces and islands of Vanuatu. It includes data on the number of affected households and individuals, as well as the geographical coordinates (latitude and longitude) of the locations. The dataset also includes details such as province, island, area, and specific location names. The data is structured in rows with columns representing status, province, island, area, location, household count, individual count, latitude, and longitude.\n",
            "Description: The dataset contains information on various governorates in Yemen, including the governorate name, code, total number of internally displaced persons (IDPs) households and individuals, as well as the total number of returnees households and individuals. The data shows varying numbers of IDPs and returnees across different governorates, with Al Hudaydah having the highest number of IDP households and individuals, while Amanat Al Asimah has the highest number of returnees households and individuals. The dataset provides insights into the displacement and returnee situation in Yemen.\n",
            "Description: The dataset contains information on various humanitarian projects implemented by different organizations in the Diffa region in February 2019. The data includes details such as the implementing partners, funding sources, start and end dates of activities, sectors of intervention, specific activities undertaken, indicators tracked, project status, targeted populations, and demographic breakdown of beneficiaries (including refugees, displaced persons, returnees, and host community members). Activities range from food security interventions to livelihood support, agricultural assistance, early recovery initiatives, and nutrition programs. The dataset also captures geographical information, including the region, department, commune, and specific locations where the projects were carried out.\n",
            "Description: The dataset contains information on humanitarian access by commune, including data such as commune codes, region, severity scores, movement restrictions, violence, access to affected populations, military presence, interference, mines, physical infrastructure, and other comments. The data also includes population demographics and average severity scores for each entry. The dataset appears to focus on access and challenges faced in different communes within the Kayes region of Mali.\n",
            "Description: The dataset contains information on targeted populations in different communes of Haiti for the years 2019-2020. It includes data on the number of people targeted, broken down by gender and age groups (children, adults, elderly). The dataset also includes administrative codes for departments and communes. The main focus appears to be on food security interventions, with details on the sectors targeted in each commune. Some communes have no data available for the targeted population.\n",
            "Description: The dataset contains information on different localities within various states, including population projections for 2018, the number of internally displaced persons (IDPs) as of December 2018, figures on protracted displacement, returnees, South Sudanese refugees and asylum seekers, affected vulnerable residents, IPC phase 4 and 3 classifications, and Global Acute Malnutrition (GAM) rates. Each row represents a different locality within a state, with data on these variables provided for each locality. The dataset appears to be structured with columns representing different categories of information and rows representing individual localities within states.\n",
            "Description: The dataset contains information on various regions in Nigeria, including the state names, local government area names, and demographic data such as the number of internally displaced persons (IDPs), returnees, and host community members. The data includes details on the targeted population, including children, adults, and elderly individuals, categorized by gender. Additionally, it provides information on the number of individuals in inaccessible areas. The dataset covers multiple regions in Nigeria and provides a comprehensive overview of the population distribution and needs in these areas.\n",
            "Description: The dataset contains population projections for different regions in the Central African Republic, disaggregated by sex and age groups. It includes information such as projected population for 2018, the number of Central African refugees, and population breakdown by gender and age categories (less than 18 years, 19 to 59 years, and 60 years and older). The data is organized by prefecture, sub-prefecture, and specific demographic details for each region.\n",
            "Processed 85.0% resources ...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on sexual violence incidents reported in various countries in 2015, including the number of staff affected, incident types, and survivor genders. The data includes countries such as Afghanistan, Belgium, Colombia, the Democratic Republic of the Congo (DRC), and Ethiopia. The incidents range from unknown types to aggressive sexual behavior, unwanted sexual comments, sexual assault, and attempted sexual assault, with female survivors being the predominant gender represented in the dataset.\n",
            "Description: The dataset contains information about various camps set up in the aftermath of the Sulawesi Earthquake in Indonesia. It includes details such as camp names, locations, access to sites, camp durations, ownership, facilities available, demographics of camp residents, health and hygiene conditions, food security, access to education, livelihood impacts, security measures, and access to various services. The camps vary in terms of status, facilities, hazards faced, and assistance received, providing a comprehensive overview of the post-disaster relief efforts in the region.\n",
            "Description: The dataset contains information on site assessments related to the Lombok earthquake in Indonesia, with columns including SSID, CAMP STATUS, DISTRICT, SUB-DISTRICT, VILLAGER, LATITUDE, LONGITUDE, LOCATION, COORDINATE, FEMALE TOTAL, MALE TOTAL, TOTAL IDPs, and TOTAL HOUSEHOLDS. Each row represents a different site assessment, with details such as the site ID, camp status, location coordinates, total number of females, males, IDPs (Internally Displaced Persons), and households. The dataset provides a snapshot of various locations in Lombok, including details on the impact of the earthquake in terms of displaced individuals and affected households.\n",
            "Description: The dataset contains information on various indicators related to incidents in Afghanistan, such as children killed and maimed, recruitment and use of children, instances of child sexual abuse, and attacks on schools and health facilities. The data includes the frequency of each indicator, the country (Afghanistan), incident verification status (verified or reported), and the year (2017). The dataset appears to be structured with columns for indicator, frequency, country, incident verification, and year, with specific data entries for each indicator.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DTM Guatemala Site Assessment Round 22.xlsx ... Error code: 400 - {'error': {'message': \"This model's maximum context length is 16385 tokens. However, your messages resulted in 21843 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}\n",
            "Description: \n",
            "Description: The dataset contains information on food prices in Lao People's Democratic Republic, with columns including date, commodity name, unit, category, price, currency, country, administrative name and code, market name and code, commodity code, and other identifiers. The data shows the retail price of palm oil in liters for various dates in 2014.\n",
            "Description: The dataset from the file \"world-humanitarian-data-and-trends-2018_full-dataset2.xlsx\" contains information related to the World Humanitarian Data and Trends report for 2018. The data includes details on funding requested, people targeted, humanitarian assistance, overall funding, capacity, reporting, inter-agency appeals, public awareness, and sector funding. The dataset also provides a table of contents, figure numbers, and Excel tab numbers for reference. The data appears to be structured with columns such as \"Unnamed: 0,\" \"WORLD HUMANITARIAN DATA AND TRENDS 2018,\" \"Unnamed: 2,\" and \"Unnamed: 3,\" with specific information listed under each category.\n",
            "Description: The dataset provided is stored in an Excel file and contains columns with information related to various aspects of a disaster event, such as the date, hour, number of tweets, tweets about affected individuals, infrastructure damage, casualties, missing/found individuals, caution/advice, donations/volunteering, sympathy/support, sentiment analysis, and images of damage. The data spans multiple time points with corresponding values for each category. The dataset appears to be structured for analysis and monitoring of disaster-related social media activity, sentiment, and impact assessment.\n",
            "Description: The dataset contains information on hospitals in Uganda, categorized by country, region, district, subcountry, and specific hospital names. It also includes the number of government facilities, private for-profit (PFP) facilities, and private not-for-profit (PNFP) facilities associated with each hospital. The data shows the distribution of these types of facilities across different regions and districts in Uganda, providing insights into the ownership structure of healthcare facilities in the country.\n",
            "Description: The dataset contains information on the estimated internally displaced population in Cameroon's North-West region as of September 2018. The data includes details such as administrative names, codes, geographical levels, status, gender, age, IDP counts, data sources, and dates. The dataset lists specific regions within the North-West area, along with corresponding IDP counts compiled by OCHA from various partners.\n",
            "Processed 86.0% resources ...\n",
            "Description: The dataset contains information on estimated population data for various Member States in the Southern African Development Community (SADC) region for the year 2018. It includes columns for Urban Population Percentage, Rural Population Percentage, Estimated Population for 2018, Estimated Urban Population for 2018, and Estimated Rural Population for 2018. The data shows the breakdown of urban and rural populations for countries such as Angola, Botswana, Comoros, DR Congo, Eswatini, Lesotho, Madagascar, Malawi, and Mauritius.\n",
            "Description: The dataset contains information on homicides in Venezuela for the year 2017, with details such as region, country, state, county, start and end dates, number of individuals affected by murder, total murders, and murders caused by police forces. The data is organized by region, state, and county within Venezuela, providing specific numbers for each category. The dataset shows a breakdown of homicides by region and highlights the number of murders caused by police forces in each area.\n",
            "Description: The dataset contains information on Ebola cases and deaths during the May 2018 outbreak in Equateur. It includes columns such as issue date, report date, total cases, total deaths, case fatality rate, confirmed cases, probable cases, suspected cases, confirmed deaths, and contacts under tracing. The data shows the progression of cases and deaths over time, with details on the number of cases, deaths, and fatality rates for each report date. The dataset also includes sources for the information provided.\n",
            "Description: The dataset contains information on sexual violence rates in different countries/territories over various years. The data includes columns for country/territory, date, number of individuals affected by sexual violence, and the corresponding rate. The dataset provides specific data points for Albania from 2003 to 2011, showing the number of individuals affected by sexual violence each year along with the rate.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Yemen Cholera Outbreak Epidemiology Data2.google sheet\n",
            "Description: The dataset contains information on IDP (Internally Displaced Persons) flow estimation in various communities within the Mohafaza region of Aleppo, Syria in July 2018. The data includes columns such as Mohafaza, PCODE, Mantika, Nahya, Community, IDP_Flow_Estimation, and provides details on the affected IDPs in each community. The dataset lists different communities, their corresponding PCODEs, and the estimated number of IDPs in each location.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset from the file \"160704_5W_HDX2.xlsx\" contains information about different organizations, their sectors, activities, locations (province, canton, and parish), and status of their projects. The data includes details such as organization name, abbreviation, sector, activity description, province, canton, parish, and project status. The organizations listed are involved in various sectors such as water, sanitation, hygiene, health, and nutrition. The dataset provides insights into the types of projects being implemented by these organizations in the specified locations, along with the current status of each project.\n",
            "Description: The dataset contains information on the number of internally displaced persons (IDPs) in various countries. The data includes countries such as Uganda, Ethiopia, Kenya, Tanzania, South Sudan, Rwanda, Burundi, Angola, and Zambia, with corresponding IDP numbers. Some countries have missing data for IDPs. Ethiopia has the highest number of IDPs with 2,800,000, followed by South Sudan with 1,900,000, and Kenya with 309,200 IDPs. Burundi has 174,011 IDPs, while the remaining countries have missing or no data available for IDPs.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset from the file \"160516_5W_ForHDX2.xlsx\" contains information about various organizations, sectors, provinces, and beneficiaries in different regions. The data includes columns such as ID, Gob, Cod.2, Organización, Sector, Provincia, Cantón, and Total beneficiaries. The organizations listed range from Agencia Adventista de Desarrollo y Recursos Asistenciales to Aldeas Infantiles SOS, with details on their activities and beneficiary counts in different areas like Manabí, Esmeraldas, Pedernales, and more. The dataset provides insights into the distribution of aid and support efforts across these regions.\n",
            "Error generating description for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Redhum-Ec 5w 2.0 Ronda 12 Versión 2061012-HDX2.xlsx ... Error code: 400 - {'error': {'message': \"Invalid 'messages[1].content': string too long. Expected a string with maximum length 1048576, but got a string with length 1983173 instead.\", 'type': 'invalid_request_error', 'param': 'messages[1].content', 'code': 'string_above_max_length'}}\n",
            "Description: \n",
            "Description: The dataset contains information on urban consumption patterns related to grid and off-grid energy sources, CO2 emissions, and expenditure in various countries. It includes data on population, different tiers of consumption, lighting and cooking types, as well as expenditure and CO2 emissions associated with grid, off-grid, nonsolid, and solid energy sources. The dataset provides details for different countries, including Afghanistan, Angola, and Albania, across different baseline and target scenarios.\n",
            "Processed 88.0% resources ...\n",
            "Description: The dataset contains survey data from the Nepal Reconstruction Survey 2018, focusing on various aspects related to housing reconstruction, food security, livelihood, and protection post-earthquake. The data includes information such as demographics (age, gender, ethnicity), housing reconstruction status, access to information and support, livelihood impact, community needs, and perceptions on protection and discrimination. Respondents provided responses to questions related to their housing reconstruction progress, food security, livelihood constraints, community needs, and perceptions of inclusion and gender equality in the recovery process. The dataset provides insights into the challenges and progress in post-earthquake recovery efforts in Nepal.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:223: UserWarning: Cell R18666 is marked as a date but the serial value 9851128965 is outside the limits for dates. The cell will be treated as an error.\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various activities implemented in different districts and VDC wards, with details such as partner organizations, funding organizations, activity types, sub-types, names, details, units, funding status, activity status, planned and reached totals, start and end dates, and comments. The data includes activities like technical assistance, training, and demonstrations related to community reconstruction and vocational training. Each entry specifies the district, VDC ward, implementing and funding partners, and GaPaNa details.\n",
            "Description: The dataset contains information on different regions in Afghanistan, including province codes, district codes, operational and organizational presence in various sectors such as ESNFI, FSAC, health, nutrition, protection, and wash. Each row represents a specific region with corresponding data on operational and organizational capacities in the mentioned sectors. The dataset provides a detailed breakdown of the presence and capacity of various organizations in different regions of Afghanistan during the specified time period.\n",
            "Description: The dataset contains information on the number of beneficiaries reached in various countries through the SOS KD WCAF outreach program in 2017. The data includes countries such as Benin, Burkina Faso, Cameroon, Cape Verde, Central African Republic, Chad, Democratic Republic of the Congo, Ivory Coast, and Equatorial Guinea, along with the corresponding number of beneficiaries reached in each country. The dataset provides a snapshot of the outreach efforts and impact of the program in these countries during the specified year.\n",
            "Description: The dataset contains information related to Ebola treatment centers, isolation wards, hospitals, and transit centers in West Africa. It includes details such as the type of facilities, color key classifications, partner involvement, and updates on the status of the centers. The data also mentions that not all health facilities may be recorded and that the information is subject to updates as more data becomes available. The dataset was last fully updated on 19/11/2014 and includes information on triage, referral, isolation, and geographical centroids of counties and towns.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake damage2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake damage2.google sheet\n",
            "Description: The dataset contains information on Zika cases in different countries/territories, including details such as the severity (confirmed, deaths, suspected), sub-regions, population, tracker update date, data publication date, EPI week, and the number of cases. The data is structured in rows with columns representing different attributes for each entry. The dataset provides insights into the spread and impact of Zika virus in various regions, with details on the number of confirmed, suspected, and death cases reported during a specific EPI week.\n",
            "Description: The dataset contains rainfall data for different administrative regions in Haiti for the month of October. It includes information such as administrative codes, names of regions, and rainfall measurements for each day in October (from the 3rd to the 6th). The data shows the total rainfall for each region during this period.\n",
            "Description: The dataset contains information on various projects in Senegal, including project names, abbreviations, total beneficiaries, direct and indirect beneficiaries, fiscal year, and domains of intervention. Projects range from financial inclusion and health outcomes to agriculture and peacebuilding. The total number of beneficiaries for each project varies, with some projects reaching over a million individuals. The dataset provides a snapshot of the impact and focus areas of different initiatives in Senegal during the specified fiscal year.\n",
            "Description: The dataset is stored in an Excel file named \"affected-schools-update-september-20172.xlsx\" and contains information about different regions in Yemen. The data includes region names in Arabic and English, with the corresponding translations. The dataset appears to have a column labeled \"Unnamed: 0\" that includes region labels and names.\n",
            "Description: The dataset contains information on tweets related to Hurricane Harvey, with columns representing date, hour, number of tweets, tweets related to affected individuals, infrastructure damage, injured or dead people, missing and found people, caution and advice, donation and volunteering, sympathy and support, other useful reports, negative sentiment, neutral sentiment, and positive sentiment. The data spans from August 25, 2017, with varying numbers of tweets recorded per hour. The dataset provides insights into the types of tweets being shared during the hurricane, including sentiment analysis and specific topics being discussed such as damage, support, and advice.\n",
            "Processed 90.0% resources ...\n",
            "Description: The dataset provided is in an Excel file format and contains information related to population estimates for different administrative zones in Mali for the period of March to August 2017. The data includes details on the estimation of populations in acute food insecurity, classifications for different administrative levels, and percentages of households affected by different phases of food insecurity. The dataset also includes information on population totals for different phases of food insecurity and projected data for the following months.\n",
            "Description: The dataset contains information on different organizations operating in Honduras, including their organization type, sector of operation, department, and department Pcode. The organizations listed include ACI and ADRA, both identified as international NGOs. The sectors of operation range from protection and education to food security and shelter/NFI. The departments mentioned include Francisco Morazan, Valle, Choluteca, and Cortes, among others, each identified by a specific Pcode.\n",
            "Description: The dataset contains information on crisis response interventions in different regions and communes in Mali during 2018. It includes details such as the type of crisis, response period, number of households and individuals reached, type of intervention, and geographical coordinates. The interventions mentioned include providing shelters, food, mosquito nets, water, sanitation, and non-food items. The data covers crises like flooding and conflicts, with various regions and communes being affected.\n",
            "Description: The dataset contains survey data from the Nepal Reconstruction Survey conducted in September 2017. It includes information on respondents' demographics such as district, age, gender, marital status, and caste/ethnicity. The survey also covers various aspects related to housing reconstruction post-earthquake, including the current status of respondents' homes, access to reconstruction support, consultation with engineers, receipt of reconstruction support, barriers faced, satisfaction with the grant dispersal process, community reconstruction needs, progress in the reconstruction process, tensions within the community, concerns related to children, exclusion or discrimination in reconstruction activities, gender equality in the reconstruction process, feedback provided to the government and NGOs on reconstruction processes. The data is structured with indicators for different questions related to housing reconstruction and protection measures.\n",
            "Description: The dataset contains information on site assessments in Burundi, including details such as site names, assessment dates, site status, GPS coordinates, administrative divisions, number of households and individuals, demographics, and security incident reporting. The data also includes information on reasons for displacement, site types, access to various services, and presence of management committees and agencies at the sites. The dataset covers multiple sites with varying characteristics and provides insights into the living conditions and needs of internally displaced persons in Burundi.\n",
            "Description: The dataset contains information on the number of people affected and displaced by floods in various districts of Somalia. It includes data such as the state, region, district name, number of people affected, number of people displaced, and population. For example, in districts like Banadir, Galmudug, and Hirshabelle, varying numbers of people have been affected and displaced by the floods. The dataset provides a snapshot of the impact of the floods on different regions and districts within Somalia.\n",
            "Description: The dataset contains information on weekly reported cases of cholera in various countries in West and Central Africa for the year 2015. The data includes columns for the ISO3 country code, country name, and weekly case counts for each week of the year. Most countries reported zero cases throughout the year, while some countries like Cote d'Ivoire and DR Congo had higher case counts. Overall, the dataset provides a snapshot of cholera incidence in the region for that year.\n",
            "Description: The dataset contains information on refugees in different regions, departments, and sites in Cameroon. It includes data on the number of female and male refugees across various age groups, as well as the total population of refugees in each location. The dataset provides details such as region names, region codes, department names, department codes, and site installations. The data is structured in a tabular format with columns representing different demographic categories and rows representing specific locations within the country.\n",
            "Description: The dataset contains information on cash activities in Nigeria from May to November 2017, with details such as date, project sector, organization, number of beneficiaries, cash disbursement in dollars, administrative region names and codes, and country information. The data shows cash disbursements for various activities related to food security, with specific beneficiary counts and amounts disbursed for each entry. The dataset appears to be structured with columns for different attributes and includes data for multiple instances of cash disbursements by different organizations in various administrative regions of Nigeria.\n",
            "Description: The dataset titled \"Refugee Population in Afghanistan 20172.xlsx\" contains information on the number of Afghan refugees returning to Afghanistan from 1 Jan to 21 Oct 2017. The data includes details such as the country of origin of refugees, province, district, and the number of male and female refugee children in different age groups. Additionally, there are notes provided regarding the source of data for specific regions.\n",
            "Processed 92.0% resources ...\n",
            "Description: The dataset contains information on displaced persons in different departments and sites within the Pool region. It includes data on the total number of men, women, individuals, households, elderly individuals, persons with disabilities, pregnant women, widows, orphans, sick individuals, infants, primary school students, secondary school students, and college students. The data is broken down by department and site, with details on the demographic composition of each location.\n",
            "Description: The dataset contains information on humanitarian aid activities in Iraq in 2015, including details such as the date of activity, organizations involved, types of aid provided (shelter, non-food items), activity status (planned or delivered), modality of aid delivery (in-kind, cash, voucher), and beneficiary descriptions. The data includes specific quantities of items distributed, such as polyethylene floor insulation, tents, mattresses, blankets, hygiene kits, and various other items. The dataset also includes information on the average cost per family, number of families reached, and total number of people assisted.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/#HXL core vocabulary list2.google sheet\n",
            "Description: The dataset contains information on 513 organizations responding to the Nepal earthquake as of May 6, 2015. It includes details such as the organization's name, type, country of origin, clusters of response activities, specific responses, capacities, website and social media links, donation links, news sources, and contact information. The organizations are involved in various sectors such as early recovery, shelter, health, WASH, nutrition, assessments, and funding. Examples of responses include providing food, shelter, medical assistance, sanitation facilities, and distributing relief materials. The dataset also includes information on partnerships, activities prior to the earthquake, and ongoing emergency responses in different districts.\n",
            "Description: The dataset from the file Mt_Agung2.csv contains information on evacuations related to the Mt. Agung volcano. It includes data on the district, date, number of evacuation sites, and number of evacuees. The data spans from September 22, 2017, to September 30, 2017, for the Badung district. The number of evacuation sites and evacuees varies each day, with the highest number of evacuees recorded on September 27, 2017, at 5879 individuals.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Unknown extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on regions, districts, communes, and fokontany in Madagascar, along with data on the number of deceased persons, injured persons, damaged houses, flooded houses, roofless houses, displaced persons, and displaced families. It also includes demographic details such as the number of individuals under 5 years old, pregnant women, handicapped individuals, and those over 60 years old. The data provides a detailed overview of the impact of various events on different areas within Madagascar, including the number of affected individuals and households.\n",
            "Description: The dataset contains information on conflict-induced displacements in Afghanistan for the year 2016. It includes details such as the newly displaced population due to conflict, compiled by OCHA sub offices based on inter-agency assessment results. The data provides information on the date of displacement, province code and name of origin and displacement, as well as district code and name of origin. The dataset is a snapshot as of June 4, 2017, with the note that the numbers are subject to change as new assessment figures become available.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various organizations, partners, clusters, sub-clusters, activity statuses, types, regions, provinces, cities/municipalities, barangays, evacuation sites, start and finish dates of activities in the Philippines as of December 2016. The activities range from protection, mine action, DRRM, food security, agriculture, livelihood, education, and emergency response management training. The dataset includes details on ongoing, completed, and planned activities conducted by organizations such as FSD, UNHCR, World Vision, Lutheran World Relief, and Save the Children in different regions and provinces of the Philippines.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DC_OP4_DANA2.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DC_OP4_DANA2.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DC_OP4_DANA2.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DC_OP4_DANA2.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Error reading /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DC_OP4_DANA2.xlsx ... Excel file format cannot be determined, you must specify an engine manually.\n",
            "Description: The dataset contains information on nutrition assistance in various countries in the Sahel region for the year 2017. It includes data on the number of people affected, in need, and targeted for assistance, as well as the percentage of targeted individuals. The countries included are Burkina Faso, Cameroon, Chad, Gambia, Mali, Mauritania, Niger, Nigeria, and Senegal. The data shows varying levels of assistance provided, with some countries reaching 100% of their targeted population while others fall short.\n",
            "Description: The dataset contains information on appeal status in the Lake Chad Basin region for the years 2014 to 2017. It includes data on countries like Cameroon and Nigeria, with details such as appeal ID, name, code, start and end dates, year, revised requirements in USD, total funding in USD, and the percentage funded. The dataset shows the funding status of different appeals over the years, highlighting the challenges in meeting the funding requirements for humanitarian efforts in the region.\n",
            "Description: The dataset contains information on displaced populations in the Lake Chad Basin region, with data for Cameroon specifically. The columns include the country name, reported location, period of reporting, total number of displaced individuals, and the type of displacement. The data shows the number of refugees in the Extrême-Nord region of Cameroon over various reporting periods from 2015 to 2016.\n",
            "Processed 94.0% resources ...\n",
            "Description: The dataset titled \"Lake Chad Basin Estimated Population 2017-02-232\" contains information on estimated populations in different regions of Cameroon as of January 1, 2017. The data includes columns for Country, Reported Location, Total Population, and AsOfDate. The regions listed include Adamaoua, Centre, Est, Extrême-Nord, Littoral, Nord, Nord-Ouest, Ouest, and Sud, with corresponding population figures. The dataset provides a snapshot of the estimated population distribution across these regions in Cameroon as of the specified date.\n",
            "Description: The dataset contains information related to activities, organizations, sectors, and demographics in Pakistan's Khyber Pakhtunkhwa and Federally Administered Tribal Areas (FATA) regions in 2016. It includes details such as activity codes, crisis types, organization funders, project descriptions, dates, administrative divisions, locations, capacities, targets, and reached populations. The activities range from skills training to community building and awareness sessions, implemented by various organizations like UNHCR. The dataset also captures data on the number of individuals reached and targeted across different demographic groups, such as adults, children, and others.\n",
            "Description: The dataset contains information on suspected, confirmed, and death cases of Zika virus in various countries, along with additional data such as cases of Congenital Syndrome and Guillain-Barre Syndrome, imported cases, and cases involving pregnant individuals. Each row represents a different country, with columns indicating the ISO3 code, country name, and corresponding data for each category. The dataset provides a snapshot of the Zika virus situation in different countries, including the number of cases and specific conditions related to the virus.\n",
            "Description: The dataset contains information on the number of houses damaged in various provinces and municipalities in the Philippines due to Typhoon Haima. It includes columns for province, municipality or city, number of houses totally damaged, number of houses partially damaged, total number of affected houses, province code, and municipality code. The data shows the extent of damage in different areas, with some municipalities reporting both totally and partially damaged houses. The dataset provides a detailed breakdown of the impact of the typhoon on housing infrastructure in the affected regions.\n",
            "Description: The dataset contains information on Zika virus cases in various countries, including suspected cases, deaths, confirmed cases, congenital syndrome, Guillain-Barré Syndrome, imported cases, and pregnant cases. The data includes columns for ISO code, country name, and specific numerical values for each category. Some entries have missing data indicated by \"NaN.\" The dataset provides a snapshot of Zika virus cases in different countries, with varying levels of reported cases and impacts.\n",
            "Description: The dataset contains information on various organizations, sectors, types of activities, and activities related to humanitarian aid and development projects. It includes details such as the organization executing the activity, the sector it falls under, the type of activity, the quantity, unit, location (province, canton, parish), status of the project, and the number of beneficiaries. The data also includes information on specific activities carried out by organizations like Acción Contra el Hambre, including support for lactating mothers, nutritional evaluations for children under 5, water and sanitation projects such as well drilling and rehabilitation, and hygiene promotion activities.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on organizations executing various activities related to water, sanitation, and hygiene in different regions of Esmeraldas, Ecuador. The data includes details such as organization names, sectors, activities, quantities, units, provinces, cantons, parishes, and status of the projects. Organizations like OPS/OMS, Cruz Roja Ecuatoriana, and World Vision are involved in activities such as water management practices, installation of water treatment plants, distribution of hygiene kits, and water chlorination. The dataset provides a snapshot of ongoing and completed projects in the region.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset contains information on various organizations, sectors, types of activities, codes, quantities, units, locations (including provinces, cantons, and parishes), types of locations, shelters, status of activities, and total number of beneficiaries. The data includes details such as organization names, sectors like Water, Sanitation, and Hygiene, specific activities undertaken by each organization, quantities of items distributed or actions taken, and the status of each activity (e.g., in execution or finalized). The dataset covers multiple entries for different organizations, each with specific details related to their humanitarian efforts in Ecuador.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake - April 2016 - Severity index2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake - April 2016 - Severity index2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake - April 2016 - Severity index2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Ecuador Earthquake - April 2016 - Severity index2.google sheet\n",
            "Description: The dataset contains information on various projects in different districts and regions in Burundi. It includes details such as reference numbers, district names, regions, P-Codes, sectors, activity types, organizations involved, project descriptions, start and end dates. The projects focus on areas like food security, WASH (Water, Sanitation, and Hygiene), education, and livelihood support. Funding sources and implementation partners are also mentioned for some projects. The projects aim to address food security needs, provide emergency assistance, promote hygiene, and support vulnerable populations in different regions.\n",
            "Description: The dataset contains information on different sectors such as CCCM, Early Recovery, Education, ETC, Food, Health, and Logistics. Each sector has counts for various attributes like Agency Acronym, Agency Full Name, Activity type, Activity Description, District, End Date, Start Date, DistPCode, VDC, VDCPCode, Other location, and Status. The data includes counts for each sector across these attributes, with some missing values indicated by NaN.\n",
            "Processed 95.0% resources ...\n",
            "Description: The dataset contains information on various projects implemented by different organizations in Somalia, focusing on sectors such as food security, WASH, education, and livelihood support. Each entry includes details such as the project reference number, districts and regions targeted, project start and end dates, as well as a description of the activities undertaken. Projects aim to address urgent needs, such as food security, livelihood recovery, and water crises, targeting vulnerable populations in different regions of Somalia. Funding sources and implementing partners are also mentioned for each project entry.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Unknown extension is not supported and will be removed\n",
            "  warn(msg)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Description: The dataset is stored in an Excel file named \"141121 LR Health Care Facilities2.xlsx\" and contains information related to health care facilities in Guinea, Liberia, and Sierra Leone. The data was collected, collated, and cleaned by Standby Task Force volunteers during September and October 2014. The dataset includes links to updated maps and is free for use in nonprofit humanitarian projects. The file also provides a link to the Standby Task Force Maps portal for further information.\n",
            "Description: The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of information. The data includes various health centers across different districts and chiefdoms in Sierra Leone, with details on their capacities, equipment, and verification status. The dataset seems to focus on health posts, health centers, and villages in Sierra Leone, providing information on their locations and operational status.\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet\n",
            "Unknown file type for /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "repr_error": "0",
              "type": "dataframe",
              "variable_name": "df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-fa95c996-b3da-4837-b75e-468ec335c534\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "      <th>Text header</th>\n",
              "      <th>Locations</th>\n",
              "      <th>Data provider</th>\n",
              "      <th>HDX dataset id</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Date created</th>\n",
              "      <th>Unnamed: 9</th>\n",
              "      <th>Hash</th>\n",
              "      <th>Data excerpt</th>\n",
              "      <th>File</th>\n",
              "      <th>URL</th>\n",
              "      <th>tag</th>\n",
              "      <th>Data description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>#affected+hh</td>\n",
              "      <td>Total IDP HH</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[319283]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "      <td>#affected</td>\n",
              "      <td>The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>#affected+ind+returnees</td>\n",
              "      <td>Total Returnees</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>true</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[587705]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "      <td>#affected</td>\n",
              "      <td>The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>#country</td>\n",
              "      <td>country_name</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#country</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>102</th>\n",
              "      <td>#geo+lat</td>\n",
              "      <td>latitude</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#geo</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103</th>\n",
              "      <td>#geo+lon</td>\n",
              "      <td>longitude</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['69.3376', '69.6258', '68.887', '68.2174', '69.1631', '70.4529', '70.1638', '69.798', '68.9114', '67.2373', '67.812']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#geo</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689531</th>\n",
              "      <td>#adm1</td>\n",
              "      <td>Province</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm1</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689533</th>\n",
              "      <td>#adm2</td>\n",
              "      <td>District</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm2</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689535</th>\n",
              "      <td>#adm3</td>\n",
              "      <td>Chiefdom</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm3</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689545</th>\n",
              "      <td>#adm1+name</td>\n",
              "      <td>Nom de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "      <td>#adm1</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689546</th>\n",
              "      <td>#adm1+code</td>\n",
              "      <td>P-code de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "      <td>#adm1</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3341 rows × 14 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fa95c996-b3da-4837-b75e-468ec335c534')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fa95c996-b3da-4837-b75e-468ec335c534 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fa95c996-b3da-4837-b75e-468ec335c534');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-740690ac-3a15-4811-b4f6-9708d3170888\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-740690ac-3a15-4811-b4f6-9708d3170888')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-740690ac-3a15-4811-b4f6-9708d3170888 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_cf45a5e4-5b4b-4cc9-9eaa-ba189cdf1e27\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_cf45a5e4-5b4b-4cc9-9eaa-ba189cdf1e27 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "        Hashtag with Attributes          Text header  \\\n",
              "1                  #affected+hh         Total IDP HH   \n",
              "8       #affected+ind+returnees      Total Returnees   \n",
              "100                    #country         country_name   \n",
              "102                    #geo+lat             latitude   \n",
              "103                    #geo+lon            longitude   \n",
              "...                         ...                  ...   \n",
              "689531                    #adm1             Province   \n",
              "689533                    #adm2             District   \n",
              "689535                    #adm3             Chiefdom   \n",
              "689545               #adm1+name     Nom de la région   \n",
              "689546               #adm1+code  P-code de la région   \n",
              "\n",
              "                                                                                          Locations  \\\n",
              "1                                                                                               COD   \n",
              "8                                                                                               COD   \n",
              "100     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "102     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "103     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "...                                                                                             ...   \n",
              "689531                                                                                          SLE   \n",
              "689533                                                                                          SLE   \n",
              "689535                                                                                          SLE   \n",
              "689545                                                                                          GIN   \n",
              "689546                                                                                          GIN   \n",
              "\n",
              "                                   Data provider  \\\n",
              "1       international-organization-for-migration   \n",
              "8       international-organization-for-migration   \n",
              "100         eth-zurich-weather-and-climate-risks   \n",
              "102         eth-zurich-weather-and-climate-risks   \n",
              "103         eth-zurich-weather-and-climate-risks   \n",
              "...                                          ...   \n",
              "689531                        standby-task-force   \n",
              "689533                        standby-task-force   \n",
              "689535                        standby-task-force   \n",
              "689545                        ipc-cluster-guinea   \n",
              "689546                        ipc-cluster-guinea   \n",
              "\n",
              "                                                                                    HDX dataset id  \\\n",
              "1       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "8       drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "100                                                                     climada-earthquake-dataset   \n",
              "102                                                                     climada-earthquake-dataset   \n",
              "103                                                                     climada-earthquake-dataset   \n",
              "...                                                                                            ...   \n",
              "689531                                                       141121-sierra-leone-health-facilities   \n",
              "689533                                                       141121-sierra-leone-health-facilities   \n",
              "689535                                                       141121-sierra-leone-health-facilities   \n",
              "689545                                                               guinea-healthcare-master-data   \n",
              "689546                                                               guinea-healthcare-master-data   \n",
              "\n",
              "                             HDX resource id Date created Unnamed: 9  \\\n",
              "1       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "8       26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       true   \n",
              "100     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "102     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "103     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "...                                      ...          ...        ...   \n",
              "689531  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689533  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689535  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "689545  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "689546  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "\n",
              "                      Hash  \\\n",
              "1       0x2cc7fd3129c0d18c   \n",
              "8       0x2cc7fd3129c0d18c   \n",
              "100     0x234606b299c1e43e   \n",
              "102     0x234606b299c1e43e   \n",
              "103     0x234606b299c1e43e   \n",
              "...                    ...   \n",
              "689531  0x5c648c638f567754   \n",
              "689533  0x5c648c638f567754   \n",
              "689535  0x5c648c638f567754   \n",
              "689545  0x47c11098111f7d90   \n",
              "689546  0x47c11098111f7d90   \n",
              "\n",
              "                                                                                                                                                                 Data excerpt  \\\n",
              "1                                                                                                                                                                    [319283]   \n",
              "8                                                                                                                                                                    [587705]   \n",
              "100     ['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']   \n",
              "102                                                 ['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']   \n",
              "103                                                    ['69.3376', '69.6258', '68.887', '68.2174', '69.1631', '70.4529', '70.1638', '69.798', '68.9114', '67.2373', '67.812']   \n",
              "...                                                                                                                                                                       ...   \n",
              "689531                                              ['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']   \n",
              "689533                                                         ['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']   \n",
              "689535                                                                               ['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']   \n",
              "689545                                                                                       ['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']   \n",
              "689546                                                                                       ['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']   \n",
              "\n",
              "                                                                                                                             File  \\\n",
              "1       /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "8       /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "100                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "102                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "103                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "...                                                                                                                           ...   \n",
              "689531                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689533                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689535                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "689545                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "689546                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "\n",
              "                                                                                                                                                                        URL  \\\n",
              "1       https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx   \n",
              "8       https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx   \n",
              "100            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "102            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "103            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "...                                                                                                                                                                     ...   \n",
              "689531   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "689533   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "689535   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "689545                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0   \n",
              "689546                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0   \n",
              "\n",
              "              tag  \\\n",
              "1       #affected   \n",
              "8       #affected   \n",
              "100      #country   \n",
              "102          #geo   \n",
              "103          #geo   \n",
              "...           ...   \n",
              "689531      #adm1   \n",
              "689533      #adm2   \n",
              "689535      #adm3   \n",
              "689545      #adm1   \n",
              "689546      #adm1   \n",
              "\n",
              "                                                                                                                                                                                               Data description  \n",
              "1       The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...  \n",
              "8       The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...  \n",
              "100     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "102     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "103     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "...                                                                                                                                                                                                         ...  \n",
              "689531  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "689533  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "689535  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "689545                                                                                                                                                                                                           \n",
              "689546                                                                                                                                                                                                           \n",
              "\n",
              "[3341 rows x 14 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def generate_data_description(data, file_name):\n",
        "    \"\"\"\n",
        "    Generate a short description of a dataset based on a summary of its content.\n",
        "\n",
        "    Args:\n",
        "        data (DataFrame): The input dataset for which a description needs to be generated.\n",
        "        file_name (str): The name of the file.\n",
        "\n",
        "    Returns:\n",
        "        str: A short description of the dataset.\n",
        "    \"\"\"\n",
        "\n",
        "    prompt = f\"\"\"\n",
        "      This data file ...\n",
        "\n",
        "      {file_name}\n",
        "\n",
        "      Has data that looks like this ...\n",
        "\n",
        "      {data.iloc[0:DATA_EXCERPT_SIZE].to_string()}\n",
        "\n",
        "      Summarize this dataset\n",
        "    \"\"\"\n",
        "\n",
        "    # Define conversation messages\n",
        "    messages = [\n",
        "        {\n",
        "            \"role\": \"system\",\n",
        "            \"content\": \"You are a helpful assistant Summarizing data into one paragraph\",\n",
        "        },\n",
        "        {\n",
        "            \"role\": \"user\",\n",
        "            \"content\": prompt\n",
        "        },\n",
        "    ]\n",
        "\n",
        "    # Request a completion (description) from the OpenAI API\n",
        "    try:\n",
        "        response = client.chat.completions.create(\n",
        "            model=DATA_SUMMARY_LLM,\n",
        "            messages=messages,\n",
        "            temperature=0,\n",
        "            max_tokens=300,\n",
        "            stop=[\"\\n\\n\"],\n",
        "        )\n",
        "    except Exception as e:\n",
        "        print(f\"Error generating description for {file_name} ... {e}\")\n",
        "        return \"\"\n",
        "\n",
        "    return response.choices[0].message.content\n",
        "\n",
        "\n",
        "def generate_data_descriptions(data_in):\n",
        "\n",
        "    data = data_in.copy()\n",
        "    data[\"Data description\"] = \"\"\n",
        "\n",
        "    unique_resources = data[\"HDX resource id\"].unique().shape[0]\n",
        "    print(f\"\\n\\nUnique resources: {unique_resources}\\n\")\n",
        "\n",
        "    count = 0\n",
        "    dataset_descriptions = {}\n",
        "    for index, row in data.iterrows():\n",
        "        resource_name = row[\"File\"].replace(f\"{LOCAL_DATA_DIR}/\", \"\")\n",
        "        resource_id = row[\"HDX resource id\"]\n",
        "        if resource_id not in dataset_descriptions:\n",
        "            # We need a try/except as not all downloads from HDX succeed\n",
        "            try:\n",
        "                if \".xlsx\" in resource_name:\n",
        "                    df = pd.read_excel(resource_name)\n",
        "                elif \".csv\" in resource_name:\n",
        "                    df = pd.read_csv(resource_name)\n",
        "                else:\n",
        "                    print(f\"Unknown file type for {resource_name}\")\n",
        "                    continue\n",
        "            except Exception as e:\n",
        "                print(f\"Error reading {resource_name} ... {e}\")\n",
        "                continue\n",
        "\n",
        "            dataset_descriptions[resource_id] = generate_data_description(\n",
        "                df, resource_name\n",
        "            )\n",
        "\n",
        "            print(f\"Description: {dataset_descriptions[resource_id]}\")\n",
        "\n",
        "            data.loc[data[\"HDX resource id\"] == resource_id, \"Data description\"] = (\n",
        "                dataset_descriptions[resource_id]\n",
        "            )\n",
        "\n",
        "            count += 1\n",
        "            if count % 10 == 0:\n",
        "                print(f\"Processed {round(count/unique_resources,2)*100}% resources ...\")\n",
        "\n",
        "    return data\n",
        "\n",
        "\n",
        "df = generate_data_descriptions(data)\n",
        "display(df)\n",
        "\n",
        "df.to_csv(f\"{LOCAL_DATA_DIR}/hxl_hash_resources_data.csv\", index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "us6Hiaf8Aiz0"
      },
      "source": [
        "## Train/Test split\n",
        "\n",
        "We have already removed a lot of duplication in our training data due to automated pipelines uploading data to HDX, next we need to split the data into train and test for model fine-tuning. Doing this with a random sample isn't a great idea because the same data provider organizations would occur in both.\n",
        "\n",
        "Most organizations are consistent in how the apply HXL tags, so having the same organization in test as in training might yield unrealistically accurate results. Therefore, we will split by data providers so that the test set only has data from providers that are not in the training data.\n",
        "\n",
        "Additionally, data provider organizations can be subsidiaries of a larger organization, for example \"ocha-iraq\" and \"ocha-libya\". It may be the case that different subsidiaries are consistent in how they tag data, so mixing data from different subsidiaries in train and test might yield unrealistically accurate results. We will therefore remove any potential subsidiaries based on organization name prefix from the test data, simulating the case of a brand-new organization will be predicting HXL tags on its data.\n",
        "\n",
        "We will also remove certain data providers from the test set because their type of data is too similar and so we would be testing the technique on predicting things (e.g. 'ourairports'), or they are part of a UN organization that will be adept of setting HXL tags and so great as training data (e.g. 'hdx', 'un-ocha')."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7yi-V0CdAiz0",
        "outputId": "098bea3d-5cb3-41e3-c3d7-2181fe63c736"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(3341, 14)\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 3341,\n  \"fields\": [\n    {\n      \"column\": \"Hashtag with Attributes\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 372,\n        \"samples\": [\n          \"#reached+m+returnees\",\n          \"#affected+injured\",\n          \"#country+code\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Text header\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 1860,\n        \"samples\": [\n          \"Returnees\",\n          \"ADM3_Baladiya_EN\",\n          \"IDPROV\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Locations\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 117,\n        \"samples\": [\n          \"AFG BGD BRA BFA KHM CAF COD ETH FRA GHA IND ITA KEN LBN LBY MLI MLT MEX MAR MMR NZL NGA PAK SOM SSD SDN CHE SYR TUR USA YEM\",\n          \"VNM\",\n          \"ALB DZA AND AIA ARG AUT AZE BHR BGD BEL BMU BOL BRA BGR KHM CAN CYM CHL CHN MAC COL CRI HRV CYP CZE DNK DOM ECU EGY EST FRO FIN FRA DEU GIB GRC GRL 33071AE7-2EF7-4B78-8309-7733991B6076 GUY HUN ISL IND IDN IRN IRL 3F5FB280-3511-4FBF-B621-3A5E59057242 ISR ITA JPN 4ACD7082-57CA-4BC2-80B8-7A465D98068C KWT LVA LBN LIE LTU LUX MDV MLT MUS MEX MCO MAR MMR NPL NLD NOR OMN PAK PAN PER POL PRT QAT C92BD69F-54C0-4D02-AD30-D33FD1CD1393 RUS SHN SAU SRB SYC SGP SVK SVN ZAF ESP LKA SWE CHE TCA TUR ARE GBR USA WORLD\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Data provider\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 109,\n        \"samples\": [\n          \"ocha-indonesia\",\n          \"wfp\",\n          \"cerf\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"HDX dataset id\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 555,\n        \"samples\": [\n          \"somalia-coronavirus-covid-19-subnational-cases\",\n          \"mali-languages\",\n          \"nutrition-in-central-african-republic\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"HDX resource id\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 555,\n        \"samples\": [\n          \"8cad612d-0245-45ae-ad25-1e9c9d82eb8b\",\n          \"32ba39ee-62a9-4b78-882d-ce7701a6113b\",\n          \"ad7efeab-75f1-4039-a643-308cf86aefc5\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Date created\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 429,\n        \"samples\": [\n          \"2016-09-06\",\n          \"2017-08-14\",\n          \"2022-11-10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unnamed: 9\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          true\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Hash\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 555,\n        \"samples\": [\n          \"0x175d2dc0c459fc9b\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Data excerpt\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2880,\n        \"samples\": [\n          \"['Flood', 'Tropical Cyclone', 'Land Slide', 'Flood', 'Flood', 'Flood', 'Epidemic', 'Flood', 'Flood', 'Flood', 'Flood']\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"File\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 555,\n        \"samples\": [\n          \"/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Somalia COVID-19 cases by location3.xlsx\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"URL\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 555,\n        \"samples\": [\n          \"https://docs.google.com/spreadsheets/d/e/2PACX-1vRTGuZDNylQKqZC7ITpHkLw-7nHvElQNtImJS7kRFXGak664t6jxDjvdVHHWkKPJ7rvwAtj6VGXrQUC/pub?output=xlsx\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tag\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 42,\n        \"samples\": [\n          \"#description\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Data description\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 520,\n        \"samples\": [\n          \"The dataset contains population data for different administrative regions in Ukraine, including the adm1_name, adm1_code, GCA/NGCA, Zone, population groups, and baseline population numbers for children, adults, elderly, Persons with Disabilities (PwD), women, and men. The data includes the total population and percentages for each population group in each region. The dataset provides detailed demographic information for regions such as Donetska and Luhanska, including the breakdown of population by age and gender.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe",
              "variable_name": "data"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-3bd11958-3548-406e-b131-960811836003\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Hashtag with Attributes</th>\n",
              "      <th>Text header</th>\n",
              "      <th>Locations</th>\n",
              "      <th>Data provider</th>\n",
              "      <th>HDX dataset id</th>\n",
              "      <th>HDX resource id</th>\n",
              "      <th>Date created</th>\n",
              "      <th>Unnamed: 9</th>\n",
              "      <th>Hash</th>\n",
              "      <th>Data excerpt</th>\n",
              "      <th>File</th>\n",
              "      <th>URL</th>\n",
              "      <th>tag</th>\n",
              "      <th>Data description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>#affected+hh</td>\n",
              "      <td>Total IDP HH</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[319283]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "      <td>#affected</td>\n",
              "      <td>The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>#affected+ind+returnees</td>\n",
              "      <td>Total Returnees</td>\n",
              "      <td>COD</td>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm</td>\n",
              "      <td>26ecc26f-74e7-46af-b450-8872dca0b63b</td>\n",
              "      <td>2023-10-16</td>\n",
              "      <td>True</td>\n",
              "      <td>0x2cc7fd3129c0d18c</td>\n",
              "      <td>[587705]</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx</td>\n",
              "      <td>#affected</td>\n",
              "      <td>The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>#country</td>\n",
              "      <td>country_name</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#country</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>#geo+lat</td>\n",
              "      <td>latitude</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#geo</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>#geo+lon</td>\n",
              "      <td>longitude</td>\n",
              "      <td>AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM</td>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>climada-earthquake-dataset</td>\n",
              "      <td>dbf9b4bd-1321-4846-b6f0-4654509d3626</td>\n",
              "      <td>2024-02-23</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0x234606b299c1e43e</td>\n",
              "      <td>['69.3376', '69.6258', '68.887', '68.2174', '69.1631', '70.4529', '70.1638', '69.798', '68.9114', '67.2373', '67.812']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv</td>\n",
              "      <td>https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv</td>\n",
              "      <td>#geo</td>\n",
              "      <td>The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3336</th>\n",
              "      <td>#adm1</td>\n",
              "      <td>Province</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm1</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3337</th>\n",
              "      <td>#adm2</td>\n",
              "      <td>District</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm2</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3338</th>\n",
              "      <td>#adm3</td>\n",
              "      <td>Chiefdom</td>\n",
              "      <td>SLE</td>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>141121-sierra-leone-health-facilities</td>\n",
              "      <td>f78dc606-04e2-4fb6-a7eb-9eb995c33f76</td>\n",
              "      <td>2014-11-01</td>\n",
              "      <td>True</td>\n",
              "      <td>0x5c648c638f567754</td>\n",
              "      <td>['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx</td>\n",
              "      <td>https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx</td>\n",
              "      <td>#adm3</td>\n",
              "      <td>The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3339</th>\n",
              "      <td>#adm1+name</td>\n",
              "      <td>Nom de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "      <td>#adm1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3340</th>\n",
              "      <td>#adm1+code</td>\n",
              "      <td>P-code de la région</td>\n",
              "      <td>GIN</td>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>guinea-healthcare-master-data</td>\n",
              "      <td>5d2531d6-c03a-449b-afdd-52c07d687679</td>\n",
              "      <td>2015-09-03</td>\n",
              "      <td>True</td>\n",
              "      <td>0x47c11098111f7d90</td>\n",
              "      <td>['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']</td>\n",
              "      <td>/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet</td>\n",
              "      <td>https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0</td>\n",
              "      <td>#adm1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3341 rows × 14 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3bd11958-3548-406e-b131-960811836003')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3bd11958-3548-406e-b131-960811836003 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3bd11958-3548-406e-b131-960811836003');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1f3563a0-6809-4f7c-9c6d-383d99a1df19\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1f3563a0-6809-4f7c-9c6d-383d99a1df19')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1f3563a0-6809-4f7c-9c6d-383d99a1df19 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_9dde3922-4457-4296-bb00-fb91207c0bca\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('data')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_9dde3922-4457-4296-bb00-fb91207c0bca button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('data');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "      Hashtag with Attributes          Text header  \\\n",
              "0                #affected+hh         Total IDP HH   \n",
              "1     #affected+ind+returnees      Total Returnees   \n",
              "2                    #country         country_name   \n",
              "3                    #geo+lat             latitude   \n",
              "4                    #geo+lon            longitude   \n",
              "...                       ...                  ...   \n",
              "3336                    #adm1             Province   \n",
              "3337                    #adm2             District   \n",
              "3338                    #adm3             Chiefdom   \n",
              "3339               #adm1+name     Nom de la région   \n",
              "3340               #adm1+code  P-code de la région   \n",
              "\n",
              "                                                                                        Locations  \\\n",
              "0                                                                                             COD   \n",
              "1                                                                                             COD   \n",
              "2     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "3     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "4     AFG BFA BDI CMR CAF TCD COL COD ETH HTI MLI MOZ MMR NER NGA SOM SSD PSE SDN SYR UKR VEN YEM   \n",
              "...                                                                                           ...   \n",
              "3336                                                                                          SLE   \n",
              "3337                                                                                          SLE   \n",
              "3338                                                                                          SLE   \n",
              "3339                                                                                          GIN   \n",
              "3340                                                                                          GIN   \n",
              "\n",
              "                                 Data provider  \\\n",
              "0     international-organization-for-migration   \n",
              "1     international-organization-for-migration   \n",
              "2         eth-zurich-weather-and-climate-risks   \n",
              "3         eth-zurich-weather-and-climate-risks   \n",
              "4         eth-zurich-weather-and-climate-risks   \n",
              "...                                        ...   \n",
              "3336                        standby-task-force   \n",
              "3337                        standby-task-force   \n",
              "3338                        standby-task-force   \n",
              "3339                        ipc-cluster-guinea   \n",
              "3340                        ipc-cluster-guinea   \n",
              "\n",
              "                                                                                  HDX dataset id  \\\n",
              "0     drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "1     drc-displacement-idps-returnees-m23-crisis-north-kivu-province-baseline-assessment-iom-dtm   \n",
              "2                                                                     climada-earthquake-dataset   \n",
              "3                                                                     climada-earthquake-dataset   \n",
              "4                                                                     climada-earthquake-dataset   \n",
              "...                                                                                          ...   \n",
              "3336                                                       141121-sierra-leone-health-facilities   \n",
              "3337                                                       141121-sierra-leone-health-facilities   \n",
              "3338                                                       141121-sierra-leone-health-facilities   \n",
              "3339                                                               guinea-healthcare-master-data   \n",
              "3340                                                               guinea-healthcare-master-data   \n",
              "\n",
              "                           HDX resource id Date created Unnamed: 9  \\\n",
              "0     26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       True   \n",
              "1     26ecc26f-74e7-46af-b450-8872dca0b63b   2023-10-16       True   \n",
              "2     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "3     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "4     dbf9b4bd-1321-4846-b6f0-4654509d3626   2024-02-23        NaN   \n",
              "...                                    ...          ...        ...   \n",
              "3336  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "3337  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "3338  f78dc606-04e2-4fb6-a7eb-9eb995c33f76   2014-11-01       True   \n",
              "3339  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "3340  5d2531d6-c03a-449b-afdd-52c07d687679   2015-09-03       True   \n",
              "\n",
              "                    Hash  \\\n",
              "0     0x2cc7fd3129c0d18c   \n",
              "1     0x2cc7fd3129c0d18c   \n",
              "2     0x234606b299c1e43e   \n",
              "3     0x234606b299c1e43e   \n",
              "4     0x234606b299c1e43e   \n",
              "...                  ...   \n",
              "3336  0x5c648c638f567754   \n",
              "3337  0x5c648c638f567754   \n",
              "3338  0x5c648c638f567754   \n",
              "3339  0x47c11098111f7d90   \n",
              "3340  0x47c11098111f7d90   \n",
              "\n",
              "                                                                                                                                                               Data excerpt  \\\n",
              "0                                                                                                                                                                  [319283]   \n",
              "1                                                                                                                                                                  [587705]   \n",
              "2     ['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']   \n",
              "3                                                 ['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']   \n",
              "4                                                    ['69.3376', '69.6258', '68.887', '68.2174', '69.1631', '70.4529', '70.1638', '69.798', '68.9114', '67.2373', '67.812']   \n",
              "...                                                                                                                                                                     ...   \n",
              "3336                                              ['Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern', 'Eastern']   \n",
              "3337                                                         ['Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema', 'Kenema']   \n",
              "3338                                                                               ['Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama', 'Dama']   \n",
              "3339                                                                                       ['Boke', 'Conakry', 'Faranah', 'Kankan', 'Kindia', 'Labe', 'Mamou', 'Nzerekore']   \n",
              "3340                                                                                       ['GIN001', 'GIN002', 'GIN003', 'GIN004', 'GIN005', 'GIN006', 'GIN007', 'GIN008']   \n",
              "\n",
              "                                                                                                                           File  \\\n",
              "0     /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "1     /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx   \n",
              "2                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "3                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "4                                    /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv   \n",
              "...                                                                                                                         ...   \n",
              "3336                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "3337                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "3338                           /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/1501 Sierra Leone Health Centers2.xlsx   \n",
              "3339                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "3340                 /content/drive/MyDrive/Colab/hxl-metadata-prediction/data/Guinea health-facility master data2.google sheet   \n",
              "\n",
              "                                                                                                                                                                      URL  \\\n",
              "0     https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx   \n",
              "1     https://data.humdata.org/dataset/3554c498-660a-45cb-ada5-86a1fbcd6056/resource/26ecc26f-74e7-46af-b450-8872dca0b63b/download/adc_27jan-12_feb_update_public_v2.xlsx   \n",
              "2            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "3            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "4            https://data.humdata.org/dataset/744f4f0b-3172-4397-9609-5ec0b9d34fcb/resource/dbf9b4bd-1321-4846-b6f0-4654509d3626/download/admin1-summaries-earthquake.csv   \n",
              "...                                                                                                                                                                   ...   \n",
              "3336   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "3337   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "3338   https://data.humdata.org/dataset/7453fb80-752b-4078-a892-d936f9846dab/resource/f78dc606-04e2-4fb6-a7eb-9eb995c33f76/download/1501-sierra-leone-health-centers.xlsx   \n",
              "3339                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0   \n",
              "3340                                                                       https://docs.google.com/spreadsheets/d/1x0MgLKLG3fxWBJ200VV5Fr67GqgSvYISefO-EYEp2wg/edit#gid=0   \n",
              "\n",
              "            tag  \\\n",
              "0     #affected   \n",
              "1     #affected   \n",
              "2      #country   \n",
              "3          #geo   \n",
              "4          #geo   \n",
              "...         ...   \n",
              "3336      #adm1   \n",
              "3337      #adm2   \n",
              "3338      #adm3   \n",
              "3339      #adm1   \n",
              "3340      #adm1   \n",
              "\n",
              "                                                                                                                                                                                             Data description  \n",
              "0     The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...  \n",
              "1     The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Dem...  \n",
              "2     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "3     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "4     The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indica...  \n",
              "...                                                                                                                                                                                                       ...  \n",
              "3336  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "3337  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "3338  The dataset contains information on health centers in Sierra Leone, including details such as center ID, status, date opened, type of center, activity, location coordinates, address, and source of...  \n",
              "3339                                                                                                                                                                                                      NaN  \n",
              "3340                                                                                                                                                                                                      NaN  \n",
              "\n",
              "[3341 rows x 14 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "data = pd.read_csv(f\"{LOCAL_DATA_DIR}/hxl_hash_resources_data.csv\")\n",
        "print(data.shape)\n",
        "display(data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xk_ZBwkPAiz0"
      },
      "source": [
        "###  Split by data provider organization\n",
        "\n",
        "On HDX, the hierarchy is ...\n",
        "\n",
        "Organization > datasets > resources > tables\n",
        "\n",
        "A random train/test split will result in data from files in a dataset being in both train and test, which would pollute the test set with very similar data to training. So we will split by organization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "kkwMFCwKAiz0",
        "outputId": "d7d0ec47-723e-4a17-9e90-c63e78204333"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "All Orgs ...\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 109,\n  \"fields\": [\n    {\n      \"column\": \"Data provider\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 109,\n        \"samples\": [\n          \"ocha-somalia\",\n          \"cpaor\",\n          \"cerf\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 80,\n        \"min\": 1,\n        \"max\": 789,\n        \"num_unique_values\": 49,\n        \"samples\": [\n          24,\n          74,\n          15\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"org_parent\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 66,\n        \"samples\": [\n          \"srsgcc\",\n          \"unrwa\",\n          \"awsd\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-6a2c1bf9-97c6-459e-8ef4-e3cdd9f0a29c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Data provider</th>\n",
              "      <th>count</th>\n",
              "      <th>org_parent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>awsd</td>\n",
              "      <td>8</td>\n",
              "      <td>awsd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>blavatnik-school-of-government-university-of-oxford</td>\n",
              "      <td>3</td>\n",
              "      <td>blavatnik</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>31</td>\n",
              "      <td>brcmapsteam</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>cbes</td>\n",
              "      <td>10</td>\n",
              "      <td>cbes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>cerf</td>\n",
              "      <td>21</td>\n",
              "      <td>cerf</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>cesvi</td>\n",
              "      <td>6</td>\n",
              "      <td>cesvi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>cfp-rco-nepal</td>\n",
              "      <td>4</td>\n",
              "      <td>cfp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>cimp</td>\n",
              "      <td>3</td>\n",
              "      <td>cimp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>cirrolytix</td>\n",
              "      <td>5</td>\n",
              "      <td>cirrolytix</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>clear</td>\n",
              "      <td>26</td>\n",
              "      <td>clear</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>cpaor</td>\n",
              "      <td>8</td>\n",
              "      <td>cpaor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>cred</td>\n",
              "      <td>7</td>\n",
              "      <td>cred</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>crs-waro</td>\n",
              "      <td>11</td>\n",
              "      <td>crs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>dalberg</td>\n",
              "      <td>4</td>\n",
              "      <td>dalberg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>dhs</td>\n",
              "      <td>28</td>\n",
              "      <td>dhs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>education-cluster-yemen</td>\n",
              "      <td>13</td>\n",
              "      <td>education</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>5</td>\n",
              "      <td>eth</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>ewipa</td>\n",
              "      <td>4</td>\n",
              "      <td>ewipa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fao</td>\n",
              "      <td>24</td>\n",
              "      <td>fao</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>fao-reowa</td>\n",
              "      <td>3</td>\n",
              "      <td>fao</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>fieldsdata</td>\n",
              "      <td>53</td>\n",
              "      <td>fieldsdata</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>global-shelter-cluster</td>\n",
              "      <td>10</td>\n",
              "      <td>global</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>hdx</td>\n",
              "      <td>135</td>\n",
              "      <td>hdx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>health-cluster</td>\n",
              "      <td>5</td>\n",
              "      <td>health</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>hera-humanitarian-emergency-response-africa</td>\n",
              "      <td>100</td>\n",
              "      <td>hera</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>hrrp-nepal</td>\n",
              "      <td>11</td>\n",
              "      <td>hrrp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>hxl</td>\n",
              "      <td>5</td>\n",
              "      <td>hxl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>iati</td>\n",
              "      <td>12</td>\n",
              "      <td>iati</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>idi</td>\n",
              "      <td>5</td>\n",
              "      <td>idi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ifrc</td>\n",
              "      <td>49</td>\n",
              "      <td>ifrc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>immap</td>\n",
              "      <td>221</td>\n",
              "      <td>immap</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>infoculture</td>\n",
              "      <td>1</td>\n",
              "      <td>infoculture</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>insecurity-insight</td>\n",
              "      <td>17</td>\n",
              "      <td>insecurity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>inter-sector-coordination-group</td>\n",
              "      <td>8</td>\n",
              "      <td>inter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>interaction</td>\n",
              "      <td>7</td>\n",
              "      <td>interaction</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>international-displacement-monitoring-centre-idmc</td>\n",
              "      <td>11</td>\n",
              "      <td>international</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>international-organization-for-migration</td>\n",
              "      <td>789</td>\n",
              "      <td>international</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>ipc</td>\n",
              "      <td>6</td>\n",
              "      <td>ipc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>104</th>\n",
              "      <td>ipc-cluster-guinea</td>\n",
              "      <td>2</td>\n",
              "      <td>ipc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>105</th>\n",
              "      <td>jcc</td>\n",
              "      <td>1</td>\n",
              "      <td>jcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>jhucsse</td>\n",
              "      <td>5</td>\n",
              "      <td>jhucsse</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>102</th>\n",
              "      <td>kenya-national-bureau-of-statistics</td>\n",
              "      <td>2</td>\n",
              "      <td>kenya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>lacso</td>\n",
              "      <td>7</td>\n",
              "      <td>lacso</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>libya-ingo-forum</td>\n",
              "      <td>5</td>\n",
              "      <td>libya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>meers</td>\n",
              "      <td>9</td>\n",
              "      <td>meers</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>moving-energy-initiative</td>\n",
              "      <td>4</td>\n",
              "      <td>moving</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>netherlands-red-cross</td>\n",
              "      <td>6</td>\n",
              "      <td>netherlands</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ocha-afghanistan</td>\n",
              "      <td>120</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>ocha-burkina</td>\n",
              "      <td>35</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>ocha-burundi</td>\n",
              "      <td>16</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>ocha-cameroon</td>\n",
              "      <td>30</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>ocha-car</td>\n",
              "      <td>46</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>ocha-chad</td>\n",
              "      <td>19</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>ocha-colombia</td>\n",
              "      <td>33</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>ocha-dr-congo</td>\n",
              "      <td>6</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>ocha-ds</td>\n",
              "      <td>9</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>ocha-eritrea</td>\n",
              "      <td>3</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>ocha-ethiopia</td>\n",
              "      <td>87</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>ocha-fiss</td>\n",
              "      <td>14</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>ocha-fts</td>\n",
              "      <td>17</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>ocha-haiti</td>\n",
              "      <td>72</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>ocha-indonesia</td>\n",
              "      <td>23</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>ocha-iraq</td>\n",
              "      <td>31</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>ocha-libya</td>\n",
              "      <td>12</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>ocha-mali</td>\n",
              "      <td>53</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>ocha-mozambique-hat</td>\n",
              "      <td>37</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>ocha-myanmar</td>\n",
              "      <td>17</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>ocha-nepal</td>\n",
              "      <td>17</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>ocha-niger</td>\n",
              "      <td>51</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ocha-nigeria</td>\n",
              "      <td>107</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>ocha-opt</td>\n",
              "      <td>8</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>ocha-pakistan</td>\n",
              "      <td>52</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ocha-philippines</td>\n",
              "      <td>81</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>ocha-roap</td>\n",
              "      <td>22</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>ocha-rolac</td>\n",
              "      <td>5</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>ocha-rosc</td>\n",
              "      <td>23</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>ocha-rosea</td>\n",
              "      <td>50</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>ocha-rowca</td>\n",
              "      <td>44</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>ocha-somalia</td>\n",
              "      <td>40</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>ocha-south-sudan</td>\n",
              "      <td>13</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>ocha-sudan</td>\n",
              "      <td>81</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79</th>\n",
              "      <td>ocha-turkey</td>\n",
              "      <td>5</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ocha-ukraine</td>\n",
              "      <td>74</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>ocha-yemen</td>\n",
              "      <td>17</td>\n",
              "      <td>ocha</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>ourairports</td>\n",
              "      <td>6</td>\n",
              "      <td>ourairports</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>qcri</td>\n",
              "      <td>10</td>\n",
              "      <td>qcri</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>rca</td>\n",
              "      <td>5</td>\n",
              "      <td>rca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103</th>\n",
              "      <td>rcpwca</td>\n",
              "      <td>2</td>\n",
              "      <td>rcpwca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>redhum</td>\n",
              "      <td>69</td>\n",
              "      <td>redhum</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>reliefweb</td>\n",
              "      <td>3</td>\n",
              "      <td>reliefweb</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107</th>\n",
              "      <td>sadc_rvaa</td>\n",
              "      <td>1</td>\n",
              "      <td>sadc_rvaa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>somalia-ngo-consortium</td>\n",
              "      <td>3</td>\n",
              "      <td>somalia</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101</th>\n",
              "      <td>soswcaf</td>\n",
              "      <td>2</td>\n",
              "      <td>soswcaf</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>srsgcc</td>\n",
              "      <td>3</td>\n",
              "      <td>srsgcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>28</td>\n",
              "      <td>standby</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>un-ocha</td>\n",
              "      <td>15</td>\n",
              "      <td>un</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>undp-human-development-reports-office</td>\n",
              "      <td>8</td>\n",
              "      <td>undp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>unesco</td>\n",
              "      <td>3</td>\n",
              "      <td>unesco</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>unhcr</td>\n",
              "      <td>5</td>\n",
              "      <td>unhcr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>unhcr-afghanistan</td>\n",
              "      <td>16</td>\n",
              "      <td>unhcr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>unicef-data</td>\n",
              "      <td>6</td>\n",
              "      <td>unicef</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>106</th>\n",
              "      <td>unicef-esaro</td>\n",
              "      <td>1</td>\n",
              "      <td>unicef</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>unicef-rdc</td>\n",
              "      <td>3</td>\n",
              "      <td>unicef</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>unodc</td>\n",
              "      <td>3</td>\n",
              "      <td>unodc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>unrwa-for-palestine-refugees-in-the-near-east</td>\n",
              "      <td>5</td>\n",
              "      <td>unrwa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>water-point-data-exchange</td>\n",
              "      <td>9</td>\n",
              "      <td>water</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>wfp</td>\n",
              "      <td>41</td>\n",
              "      <td>wfp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>world-bank-group</td>\n",
              "      <td>4</td>\n",
              "      <td>world</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>world-health-organization</td>\n",
              "      <td>10</td>\n",
              "      <td>world</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6a2c1bf9-97c6-459e-8ef4-e3cdd9f0a29c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6a2c1bf9-97c6-459e-8ef4-e3cdd9f0a29c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6a2c1bf9-97c6-459e-8ef4-e3cdd9f0a29c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0b2391df-3004-4918-93b7-98fac7278592\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0b2391df-3004-4918-93b7-98fac7278592')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0b2391df-3004-4918-93b7-98fac7278592 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                           Data provider  count     org_parent\n",
              "62                                                  awsd      8           awsd\n",
              "100  blavatnik-school-of-government-university-of-oxford      3      blavatnik\n",
              "25                                           brcmapsteam     31    brcmapsteam\n",
              "55                                                  cbes     10           cbes\n",
              "35                                                  cerf     21           cerf\n",
              "70                                                 cesvi      6          cesvi\n",
              "89                                         cfp-rco-nepal      4            cfp\n",
              "94                                                  cimp      3           cimp\n",
              "75                                            cirrolytix      5     cirrolytix\n",
              "30                                                 clear     26          clear\n",
              "64                                                 cpaor      8          cpaor\n",
              "67                                                  cred      7           cred\n",
              "50                                              crs-waro     11            crs\n",
              "90                                               dalberg      4        dalberg\n",
              "28                                                   dhs     28            dhs\n",
              "47                               education-cluster-yemen     13      education\n",
              "74                  eth-zurich-weather-and-climate-risks      5            eth\n",
              "86                                                 ewipa      4          ewipa\n",
              "31                                                   fao     24            fao\n",
              "96                                             fao-reowa      3            fao\n",
              "13                                            fieldsdata     53     fieldsdata\n",
              "54                                global-shelter-cluster     10         global\n",
              "2                                                    hdx    135            hdx\n",
              "76                                        health-cluster      5         health\n",
              "5            hera-humanitarian-emergency-response-africa    100           hera\n",
              "52                                            hrrp-nepal     11           hrrp\n",
              "81                                                   hxl      5            hxl\n",
              "48                                                  iati     12           iati\n",
              "82                                                   idi      5            idi\n",
              "17                                                  ifrc     49           ifrc\n",
              "1                                                  immap    221          immap\n",
              "108                                          infoculture      1    infoculture\n",
              "37                                    insecurity-insight     17     insecurity\n",
              "63                       inter-sector-coordination-group      8          inter\n",
              "65                                           interaction      7    interaction\n",
              "51     international-displacement-monitoring-centre-idmc     11  international\n",
              "0               international-organization-for-migration    789  international\n",
              "73                                                   ipc      6            ipc\n",
              "104                                   ipc-cluster-guinea      2            ipc\n",
              "105                                                  jcc      1            jcc\n",
              "84                                               jhucsse      5        jhucsse\n",
              "102                  kenya-national-bureau-of-statistics      2          kenya\n",
              "66                                                 lacso      7          lacso\n",
              "83                                      libya-ingo-forum      5          libya\n",
              "59                                                 meers      9          meers\n",
              "88                              moving-energy-initiative      4         moving\n",
              "69                                 netherlands-red-cross      6    netherlands\n",
              "3                                       ocha-afghanistan    120           ocha\n",
              "23                                          ocha-burkina     35           ocha\n",
              "43                                          ocha-burundi     16           ocha\n",
              "27                                         ocha-cameroon     30           ocha\n",
              "18                                              ocha-car     46           ocha\n",
              "36                                             ocha-chad     19           ocha\n",
              "24                                         ocha-colombia     33           ocha\n",
              "68                                         ocha-dr-congo      6           ocha\n",
              "58                                               ocha-ds      9           ocha\n",
              "95                                          ocha-eritrea      3           ocha\n",
              "6                                          ocha-ethiopia     87           ocha\n",
              "45                                             ocha-fiss     14           ocha\n",
              "38                                              ocha-fts     17           ocha\n",
              "10                                            ocha-haiti     72           ocha\n",
              "33                                        ocha-indonesia     23           ocha\n",
              "26                                             ocha-iraq     31           ocha\n",
              "49                                            ocha-libya     12           ocha\n",
              "12                                             ocha-mali     53           ocha\n",
              "22                                   ocha-mozambique-hat     37           ocha\n",
              "41                                          ocha-myanmar     17           ocha\n",
              "40                                            ocha-nepal     17           ocha\n",
              "15                                            ocha-niger     51           ocha\n",
              "4                                           ocha-nigeria    107           ocha\n",
              "61                                              ocha-opt      8           ocha\n",
              "14                                         ocha-pakistan     52           ocha\n",
              "8                                       ocha-philippines     81           ocha\n",
              "34                                             ocha-roap     22           ocha\n",
              "85                                            ocha-rolac      5           ocha\n",
              "32                                             ocha-rosc     23           ocha\n",
              "16                                            ocha-rosea     50           ocha\n",
              "19                                            ocha-rowca     44           ocha\n",
              "21                                          ocha-somalia     40           ocha\n",
              "46                                      ocha-south-sudan     13           ocha\n",
              "7                                             ocha-sudan     81           ocha\n",
              "79                                           ocha-turkey      5           ocha\n",
              "9                                           ocha-ukraine     74           ocha\n",
              "39                                            ocha-yemen     17           ocha\n",
              "71                                           ourairports      6    ourairports\n",
              "53                                                  qcri     10           qcri\n",
              "80                                                   rca      5            rca\n",
              "103                                               rcpwca      2         rcpwca\n",
              "11                                                redhum     69         redhum\n",
              "99                                             reliefweb      3      reliefweb\n",
              "107                                            sadc_rvaa      1      sadc_rvaa\n",
              "98                                somalia-ngo-consortium      3        somalia\n",
              "101                                              soswcaf      2        soswcaf\n",
              "97                                                srsgcc      3         srsgcc\n",
              "29                                    standby-task-force     28        standby\n",
              "44                                               un-ocha     15             un\n",
              "60                 undp-human-development-reports-office      8           undp\n",
              "91                                                unesco      3         unesco\n",
              "77                                                 unhcr      5          unhcr\n",
              "42                                     unhcr-afghanistan     16          unhcr\n",
              "72                                           unicef-data      6         unicef\n",
              "106                                         unicef-esaro      1         unicef\n",
              "93                                            unicef-rdc      3         unicef\n",
              "92                                                 unodc      3          unodc\n",
              "78         unrwa-for-palestine-refugees-in-the-near-east      5          unrwa\n",
              "57                             water-point-data-exchange      9          water\n",
              "20                                                   wfp     41            wfp\n",
              "87                                      world-bank-group      4          world\n",
              "56                             world-health-organization     10          world"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Orgs which don't seem to be subsidiaries ...\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 59,\n  \"fields\": [\n    {\n      \"column\": \"Data provider\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 59,\n        \"samples\": [\n          \"immap\",\n          \"ifrc\",\n          \"cirrolytix\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 35,\n        \"min\": 1,\n        \"max\": 221,\n        \"num_unique_values\": 26,\n        \"samples\": [\n          28,\n          10,\n          221\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"org_parent\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 59,\n        \"samples\": [\n          \"immap\",\n          \"ifrc\",\n          \"cirrolytix\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-bf6ccf8b-f599-4d2c-9177-9c1b65bfc916\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Data provider</th>\n",
              "      <th>count</th>\n",
              "      <th>org_parent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>immap</td>\n",
              "      <td>221</td>\n",
              "      <td>immap</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>hdx</td>\n",
              "      <td>135</td>\n",
              "      <td>hdx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>hera-humanitarian-emergency-response-africa</td>\n",
              "      <td>100</td>\n",
              "      <td>hera</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>redhum</td>\n",
              "      <td>69</td>\n",
              "      <td>redhum</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>fieldsdata</td>\n",
              "      <td>53</td>\n",
              "      <td>fieldsdata</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ifrc</td>\n",
              "      <td>49</td>\n",
              "      <td>ifrc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>wfp</td>\n",
              "      <td>41</td>\n",
              "      <td>wfp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>31</td>\n",
              "      <td>brcmapsteam</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>dhs</td>\n",
              "      <td>28</td>\n",
              "      <td>dhs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>standby-task-force</td>\n",
              "      <td>28</td>\n",
              "      <td>standby</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>clear</td>\n",
              "      <td>26</td>\n",
              "      <td>clear</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>cerf</td>\n",
              "      <td>21</td>\n",
              "      <td>cerf</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>insecurity-insight</td>\n",
              "      <td>17</td>\n",
              "      <td>insecurity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>un-ocha</td>\n",
              "      <td>15</td>\n",
              "      <td>un</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>education-cluster-yemen</td>\n",
              "      <td>13</td>\n",
              "      <td>education</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>iati</td>\n",
              "      <td>12</td>\n",
              "      <td>iati</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>crs-waro</td>\n",
              "      <td>11</td>\n",
              "      <td>crs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>hrrp-nepal</td>\n",
              "      <td>11</td>\n",
              "      <td>hrrp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>qcri</td>\n",
              "      <td>10</td>\n",
              "      <td>qcri</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>global-shelter-cluster</td>\n",
              "      <td>10</td>\n",
              "      <td>global</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>cbes</td>\n",
              "      <td>10</td>\n",
              "      <td>cbes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>water-point-data-exchange</td>\n",
              "      <td>9</td>\n",
              "      <td>water</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>meers</td>\n",
              "      <td>9</td>\n",
              "      <td>meers</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>undp-human-development-reports-office</td>\n",
              "      <td>8</td>\n",
              "      <td>undp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>awsd</td>\n",
              "      <td>8</td>\n",
              "      <td>awsd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>inter-sector-coordination-group</td>\n",
              "      <td>8</td>\n",
              "      <td>inter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>cpaor</td>\n",
              "      <td>8</td>\n",
              "      <td>cpaor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>interaction</td>\n",
              "      <td>7</td>\n",
              "      <td>interaction</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>lacso</td>\n",
              "      <td>7</td>\n",
              "      <td>lacso</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>cred</td>\n",
              "      <td>7</td>\n",
              "      <td>cred</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>netherlands-red-cross</td>\n",
              "      <td>6</td>\n",
              "      <td>netherlands</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>cesvi</td>\n",
              "      <td>6</td>\n",
              "      <td>cesvi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>ourairports</td>\n",
              "      <td>6</td>\n",
              "      <td>ourairports</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>eth-zurich-weather-and-climate-risks</td>\n",
              "      <td>5</td>\n",
              "      <td>eth</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>cirrolytix</td>\n",
              "      <td>5</td>\n",
              "      <td>cirrolytix</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>health-cluster</td>\n",
              "      <td>5</td>\n",
              "      <td>health</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>unrwa-for-palestine-refugees-in-the-near-east</td>\n",
              "      <td>5</td>\n",
              "      <td>unrwa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>rca</td>\n",
              "      <td>5</td>\n",
              "      <td>rca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>hxl</td>\n",
              "      <td>5</td>\n",
              "      <td>hxl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>idi</td>\n",
              "      <td>5</td>\n",
              "      <td>idi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>libya-ingo-forum</td>\n",
              "      <td>5</td>\n",
              "      <td>libya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>jhucsse</td>\n",
              "      <td>5</td>\n",
              "      <td>jhucsse</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>ewipa</td>\n",
              "      <td>4</td>\n",
              "      <td>ewipa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>moving-energy-initiative</td>\n",
              "      <td>4</td>\n",
              "      <td>moving</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>cfp-rco-nepal</td>\n",
              "      <td>4</td>\n",
              "      <td>cfp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>dalberg</td>\n",
              "      <td>4</td>\n",
              "      <td>dalberg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>unesco</td>\n",
              "      <td>3</td>\n",
              "      <td>unesco</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>unodc</td>\n",
              "      <td>3</td>\n",
              "      <td>unodc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>cimp</td>\n",
              "      <td>3</td>\n",
              "      <td>cimp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>srsgcc</td>\n",
              "      <td>3</td>\n",
              "      <td>srsgcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>somalia-ngo-consortium</td>\n",
              "      <td>3</td>\n",
              "      <td>somalia</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>reliefweb</td>\n",
              "      <td>3</td>\n",
              "      <td>reliefweb</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>blavatnik-school-of-government-university-of-oxford</td>\n",
              "      <td>3</td>\n",
              "      <td>blavatnik</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101</th>\n",
              "      <td>soswcaf</td>\n",
              "      <td>2</td>\n",
              "      <td>soswcaf</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>102</th>\n",
              "      <td>kenya-national-bureau-of-statistics</td>\n",
              "      <td>2</td>\n",
              "      <td>kenya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103</th>\n",
              "      <td>rcpwca</td>\n",
              "      <td>2</td>\n",
              "      <td>rcpwca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>105</th>\n",
              "      <td>jcc</td>\n",
              "      <td>1</td>\n",
              "      <td>jcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107</th>\n",
              "      <td>sadc_rvaa</td>\n",
              "      <td>1</td>\n",
              "      <td>sadc_rvaa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>infoculture</td>\n",
              "      <td>1</td>\n",
              "      <td>infoculture</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bf6ccf8b-f599-4d2c-9177-9c1b65bfc916')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bf6ccf8b-f599-4d2c-9177-9c1b65bfc916 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bf6ccf8b-f599-4d2c-9177-9c1b65bfc916');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2e130092-7201-425b-ab4b-97f30957a614\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2e130092-7201-425b-ab4b-97f30957a614')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2e130092-7201-425b-ab4b-97f30957a614 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                           Data provider  count   org_parent\n",
              "1                                                  immap    221        immap\n",
              "2                                                    hdx    135          hdx\n",
              "5            hera-humanitarian-emergency-response-africa    100         hera\n",
              "11                                                redhum     69       redhum\n",
              "13                                            fieldsdata     53   fieldsdata\n",
              "17                                                  ifrc     49         ifrc\n",
              "20                                                   wfp     41          wfp\n",
              "25                                           brcmapsteam     31  brcmapsteam\n",
              "28                                                   dhs     28          dhs\n",
              "29                                    standby-task-force     28      standby\n",
              "30                                                 clear     26        clear\n",
              "35                                                  cerf     21         cerf\n",
              "37                                    insecurity-insight     17   insecurity\n",
              "44                                               un-ocha     15           un\n",
              "47                               education-cluster-yemen     13    education\n",
              "48                                                  iati     12         iati\n",
              "50                                              crs-waro     11          crs\n",
              "52                                            hrrp-nepal     11         hrrp\n",
              "53                                                  qcri     10         qcri\n",
              "54                                global-shelter-cluster     10       global\n",
              "55                                                  cbes     10         cbes\n",
              "57                             water-point-data-exchange      9        water\n",
              "59                                                 meers      9        meers\n",
              "60                 undp-human-development-reports-office      8         undp\n",
              "62                                                  awsd      8         awsd\n",
              "63                       inter-sector-coordination-group      8        inter\n",
              "64                                                 cpaor      8        cpaor\n",
              "65                                           interaction      7  interaction\n",
              "66                                                 lacso      7        lacso\n",
              "67                                                  cred      7         cred\n",
              "69                                 netherlands-red-cross      6  netherlands\n",
              "70                                                 cesvi      6        cesvi\n",
              "71                                           ourairports      6  ourairports\n",
              "74                  eth-zurich-weather-and-climate-risks      5          eth\n",
              "75                                            cirrolytix      5   cirrolytix\n",
              "76                                        health-cluster      5       health\n",
              "78         unrwa-for-palestine-refugees-in-the-near-east      5        unrwa\n",
              "80                                                   rca      5          rca\n",
              "81                                                   hxl      5          hxl\n",
              "82                                                   idi      5          idi\n",
              "83                                      libya-ingo-forum      5        libya\n",
              "84                                               jhucsse      5      jhucsse\n",
              "86                                                 ewipa      4        ewipa\n",
              "88                              moving-energy-initiative      4       moving\n",
              "89                                         cfp-rco-nepal      4          cfp\n",
              "90                                               dalberg      4      dalberg\n",
              "91                                                unesco      3       unesco\n",
              "92                                                 unodc      3        unodc\n",
              "94                                                  cimp      3         cimp\n",
              "97                                                srsgcc      3       srsgcc\n",
              "98                                somalia-ngo-consortium      3      somalia\n",
              "99                                             reliefweb      3    reliefweb\n",
              "100  blavatnik-school-of-government-university-of-oxford      3    blavatnik\n",
              "101                                              soswcaf      2      soswcaf\n",
              "102                  kenya-national-bureau-of-statistics      2        kenya\n",
              "103                                               rcpwca      2       rcpwca\n",
              "105                                                  jcc      1          jcc\n",
              "107                                            sadc_rvaa      1    sadc_rvaa\n",
              "108                                          infoculture      1  infoculture"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train orgs: ['ocha-roap', 'cesvi', 'ocha-ds', 'ocha-afghanistan', 'cerf', 'unicef-rdc', 'standby-task-force', 'ocha-cameroon', 'international-organization-for-migration', 'ocha-myanmar', 'unicef-esaro', 'ocha-nigeria', 'ocha-car', 'ewipa', 'fao', 'water-point-data-exchange', 'ocha-indonesia', 'ocha-haiti', 'ocha-colombia', 'ocha-yemen', 'ocha-sudan', 'international-displacement-monitoring-centre-idmc', 'ocha-nepal', 'ocha-rowca', 'immap', 'ocha-pakistan', 'jhucsse', 'somalia-ngo-consortium', 'crs-waro', 'soswcaf', 'kenya-national-bureau-of-statistics', 'ocha-rolac', 'dalberg', 'ocha-chad', 'awsd', 'ocha-rosea', 'ocha-somalia', 'ocha-niger', 'ocha-turkey', 'eth-zurich-weather-and-climate-risks', 'sadc_rvaa', 'unhcr', 'interaction', 'ocha-fiss', 'ocha-opt', 'ocha-ukraine', 'unhcr-afghanistan', 'unicef-data', 'libya-ingo-forum', 'ocha-iraq', 'cirrolytix', 'ocha-burundi', 'ocha-mali', 'cfp-rco-nepal', 'ocha-rosc', 'global-shelter-cluster', 'cbes', 'ocha-libya', 'clear', 'ocha-fts', 'inter-sector-coordination-group', 'unrwa-for-palestine-refugees-in-the-near-east', 'ocha-ethiopia', 'wfp', 'ourairports', 'world-bank-group', 'reliefweb', 'qcri', 'undp-human-development-reports-office', 'ipc', 'world-health-organization', 'idi', 'ocha-mozambique-hat', 'fao-reowa', 'ocha-eritrea', 'ocha-burkina', 'ocha-philippines', 'ocha-dr-congo', 'un-ocha', 'ocha-south-sudan', 'ipc-cluster-guinea', 'hdx']\n",
            "Test orgs: ['moving-energy-initiative', 'unesco', 'infoculture', 'dhs', 'rcpwca', 'cimp', 'health-cluster', 'moving-energy-initiative', 'fieldsdata', 'hrrp-nepal', 'cpaor', 'cpaor', 'hxl', 'unodc', 'lacso', 'netherlands-red-cross', 'hxl', 'brcmapsteam', 'hera-humanitarian-emergency-response-africa', 'unodc', 'blavatnik-school-of-government-university-of-oxford', 'hxl', 'redhum', 'insecurity-insight', 'meers', 'blavatnik-school-of-government-university-of-oxford', 'health-cluster', 'jcc', 'cred', 'hera-humanitarian-emergency-response-africa', 'redhum', 'iati', 'srsgcc', 'ifrc', 'rca', 'education-cluster-yemen', 'dhs']\n",
            "These are the test orgs and their data counts ...\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 27,\n  \"fields\": [\n    {\n      \"column\": \"Data provider\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 27,\n        \"samples\": [\n          \"iati\",\n          \"lacso\",\n          \"hrrp-nepal\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 24,\n        \"min\": 1,\n        \"max\": 100,\n        \"num_unique_values\": 19,\n        \"samples\": [\n          100,\n          28,\n          8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"org_parent\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 27,\n        \"samples\": [\n          \"iati\",\n          \"lacso\",\n          \"hrrp\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-07352aae-118a-4cd3-bcee-785b77ee494c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Data provider</th>\n",
              "      <th>count</th>\n",
              "      <th>org_parent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>hera-humanitarian-emergency-response-africa</td>\n",
              "      <td>100</td>\n",
              "      <td>hera</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>redhum</td>\n",
              "      <td>69</td>\n",
              "      <td>redhum</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>fieldsdata</td>\n",
              "      <td>53</td>\n",
              "      <td>fieldsdata</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ifrc</td>\n",
              "      <td>49</td>\n",
              "      <td>ifrc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>brcmapsteam</td>\n",
              "      <td>31</td>\n",
              "      <td>brcmapsteam</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>dhs</td>\n",
              "      <td>28</td>\n",
              "      <td>dhs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>insecurity-insight</td>\n",
              "      <td>17</td>\n",
              "      <td>insecurity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>education-cluster-yemen</td>\n",
              "      <td>13</td>\n",
              "      <td>education</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>iati</td>\n",
              "      <td>12</td>\n",
              "      <td>iati</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>hrrp-nepal</td>\n",
              "      <td>11</td>\n",
              "      <td>hrrp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>meers</td>\n",
              "      <td>9</td>\n",
              "      <td>meers</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>cpaor</td>\n",
              "      <td>8</td>\n",
              "      <td>cpaor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>cred</td>\n",
              "      <td>7</td>\n",
              "      <td>cred</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>lacso</td>\n",
              "      <td>7</td>\n",
              "      <td>lacso</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>netherlands-red-cross</td>\n",
              "      <td>6</td>\n",
              "      <td>netherlands</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>health-cluster</td>\n",
              "      <td>5</td>\n",
              "      <td>health</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>rca</td>\n",
              "      <td>5</td>\n",
              "      <td>rca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>hxl</td>\n",
              "      <td>5</td>\n",
              "      <td>hxl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>moving-energy-initiative</td>\n",
              "      <td>4</td>\n",
              "      <td>moving</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>unesco</td>\n",
              "      <td>3</td>\n",
              "      <td>unesco</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>unodc</td>\n",
              "      <td>3</td>\n",
              "      <td>unodc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>cimp</td>\n",
              "      <td>3</td>\n",
              "      <td>cimp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>srsgcc</td>\n",
              "      <td>3</td>\n",
              "      <td>srsgcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>blavatnik-school-of-government-university-of-oxford</td>\n",
              "      <td>3</td>\n",
              "      <td>blavatnik</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103</th>\n",
              "      <td>rcpwca</td>\n",
              "      <td>2</td>\n",
              "      <td>rcpwca</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>105</th>\n",
              "      <td>jcc</td>\n",
              "      <td>1</td>\n",
              "      <td>jcc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>infoculture</td>\n",
              "      <td>1</td>\n",
              "      <td>infoculture</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-07352aae-118a-4cd3-bcee-785b77ee494c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-07352aae-118a-4cd3-bcee-785b77ee494c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-07352aae-118a-4cd3-bcee-785b77ee494c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-285c5b06-b683-4209-b4b3-b1b450786d31\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-285c5b06-b683-4209-b4b3-b1b450786d31')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-285c5b06-b683-4209-b4b3-b1b450786d31 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                           Data provider  count   org_parent\n",
              "5            hera-humanitarian-emergency-response-africa    100         hera\n",
              "11                                                redhum     69       redhum\n",
              "13                                            fieldsdata     53   fieldsdata\n",
              "17                                                  ifrc     49         ifrc\n",
              "25                                           brcmapsteam     31  brcmapsteam\n",
              "28                                                   dhs     28          dhs\n",
              "37                                    insecurity-insight     17   insecurity\n",
              "47                               education-cluster-yemen     13    education\n",
              "48                                                  iati     12         iati\n",
              "52                                            hrrp-nepal     11         hrrp\n",
              "59                                                 meers      9        meers\n",
              "64                                                 cpaor      8        cpaor\n",
              "67                                                  cred      7         cred\n",
              "66                                                 lacso      7        lacso\n",
              "69                                 netherlands-red-cross      6  netherlands\n",
              "76                                        health-cluster      5       health\n",
              "80                                                   rca      5          rca\n",
              "81                                                   hxl      5          hxl\n",
              "88                              moving-energy-initiative      4       moving\n",
              "91                                                unesco      3       unesco\n",
              "92                                                 unodc      3        unodc\n",
              "94                                                  cimp      3         cimp\n",
              "97                                                srsgcc      3       srsgcc\n",
              "100  blavatnik-school-of-government-university-of-oxford      3    blavatnik\n",
              "103                                               rcpwca      2       rcpwca\n",
              "105                                                  jcc      1          jcc\n",
              "108                                          infoculture      1  infoculture"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Train orgs: ['awsd' 'cbes' 'cerf' 'cesvi' 'cfp-rco-nepal' 'cirrolytix' 'clear'\n",
            " 'crs-waro' 'dalberg' 'eth-zurich-weather-and-climate-risks' 'ewipa' 'fao'\n",
            " 'fao-reowa' 'global-shelter-cluster' 'hdx' 'idi' 'immap'\n",
            " 'inter-sector-coordination-group' 'interaction'\n",
            " 'international-displacement-monitoring-centre-idmc'\n",
            " 'international-organization-for-migration' 'ipc' 'ipc-cluster-guinea'\n",
            " 'jhucsse' 'kenya-national-bureau-of-statistics' 'libya-ingo-forum'\n",
            " 'ocha-afghanistan' 'ocha-burkina' 'ocha-burundi' 'ocha-cameroon'\n",
            " 'ocha-car' 'ocha-chad' 'ocha-colombia' 'ocha-dr-congo' 'ocha-ds'\n",
            " 'ocha-eritrea' 'ocha-ethiopia' 'ocha-fiss' 'ocha-fts' 'ocha-haiti'\n",
            " 'ocha-indonesia' 'ocha-iraq' 'ocha-libya' 'ocha-mali'\n",
            " 'ocha-mozambique-hat' 'ocha-myanmar' 'ocha-nepal' 'ocha-niger'\n",
            " 'ocha-nigeria' 'ocha-opt' 'ocha-pakistan' 'ocha-philippines' 'ocha-roap'\n",
            " 'ocha-rolac' 'ocha-rosc' 'ocha-rosea' 'ocha-rowca' 'ocha-somalia'\n",
            " 'ocha-south-sudan' 'ocha-sudan' 'ocha-turkey' 'ocha-ukraine' 'ocha-yemen'\n",
            " 'ourairports' 'qcri' 'reliefweb' 'sadc_rvaa' 'somalia-ngo-consortium'\n",
            " 'soswcaf' 'standby-task-force' 'un-ocha'\n",
            " 'undp-human-development-reports-office' 'unhcr' 'unhcr-afghanistan'\n",
            " 'unicef-data' 'unicef-esaro' 'unicef-rdc'\n",
            " 'unrwa-for-palestine-refugees-in-the-near-east'\n",
            " 'water-point-data-exchange' 'wfp' 'world-bank-group'\n",
            " 'world-health-organization']\n",
            "\n",
            "Test orgs: ['blavatnik-school-of-government-university-of-oxford' 'brcmapsteam'\n",
            " 'cimp' 'cpaor' 'cred' 'dhs' 'education-cluster-yemen' 'fieldsdata'\n",
            " 'health-cluster' 'hera-humanitarian-emergency-response-africa'\n",
            " 'hrrp-nepal' 'hxl' 'iati' 'ifrc' 'infoculture' 'insecurity-insight' 'jcc'\n",
            " 'lacso' 'meers' 'moving-energy-initiative' 'netherlands-red-cross' 'rca'\n",
            " 'rcpwca' 'redhum' 'srsgcc' 'unesco' 'unodc']\n",
            "\n",
            "Train column data: (2883, 15)\n",
            "Test column data: (458, 15)\n"
          ]
        }
      ],
      "source": [
        "def split_data(column_data, provider_col, test_size=0.2, random_state=42):\n",
        "    \"\"\"\n",
        "    Perform train-test split on datasets, print information, and return X_train and X_test.\n",
        "\n",
        "    The split is done by organizations, to try and avoid the situation where an org provides\n",
        "    similar data files. Also, we exclude orgs which are subsidiaries from the test set, eg ocha-*\n",
        "    as presumably each subsid will provide similar data. The aim is that the test set is new.\n",
        "\n",
        "    Parameters:\n",
        "    - column_data (pd.DataFrame): DataFrame containing column data.\n",
        "    - provider_col (string): Name of column holding data providers.\n",
        "    - test_size (float): The proportion of the dataset to include in the test split.\n",
        "    - random_state (int): Seed for random number generation.\n",
        "\n",
        "    Returns:\n",
        "    - pd.DataFrame, pd.DataFrame: X_train, X_test\n",
        "    \"\"\"\n",
        "\n",
        "    orgs_df = (\n",
        "        column_data.groupby(provider_col)[provider_col]\n",
        "        .count()\n",
        "        .sort_values(ascending=False)\n",
        "    )\n",
        "    orgs_df = (\n",
        "        column_data.groupby(provider_col)[provider_col]\n",
        "        .count()\n",
        "        .sort_values(ascending=False)\n",
        "        .reset_index(name=\"count\")\n",
        "    )\n",
        "    all_orgs = orgs_df[provider_col].unique()\n",
        "\n",
        "    # Split orgs to get 'Parent', eg 'ocha-*' -> 'ocha'\n",
        "    orgs_df[\"org_parent\"] = orgs_df[provider_col].str.split(\"-\").str[0]\n",
        "\n",
        "    # Display, sorted by provider_col\n",
        "    print(\"All Orgs ...\")\n",
        "    display(orgs_df.sort_values(by=provider_col))\n",
        "\n",
        "    # Count occurrences of each 'org_parent'\n",
        "    org_parent_counts = orgs_df[\"org_parent\"].value_counts().reset_index(name=\"count\")\n",
        "\n",
        "    # Filter to keep only those occurring once\n",
        "    org_parents_single_occurrence = org_parent_counts[org_parent_counts[\"count\"] == 1]\n",
        "\n",
        "    # Get the 'org_parent' values that occur only once\n",
        "    single_occurrence_org_parents = org_parents_single_occurrence[\"org_parent\"].tolist()\n",
        "\n",
        "    # Filter the original DataFrame to keep rows where 'org_parent' occurs only once\n",
        "    org_parents_unique = orgs_df[\n",
        "        orgs_df[\"org_parent\"].isin(single_occurrence_org_parents)\n",
        "    ]\n",
        "\n",
        "    print(\"\\nOrgs which don't seem to be subsidiaries ...\\n\")\n",
        "    display(org_parents_unique)\n",
        "\n",
        "    single_entities = list(org_parents_unique[provider_col].unique())\n",
        "\n",
        "    # Remove 'hdx' from single_entities, not good for testing as it's the folks that made HXL! Also some monolithic orgs with very similar data\n",
        "    single_entities = [\n",
        "        x for x in single_entities if not x in [\"hdx\", \"ourairports\", \"un-ocha\"]\n",
        "    ]\n",
        "\n",
        "    single_entities.sort()\n",
        "\n",
        "    # Sample single-subsid orgs\n",
        "    sample_size = int(column_data.shape[0] * test_size) - 1\n",
        "\n",
        "    # Sample orgs and total rows they have to arrive at fraction specified for test\n",
        "    total_count = 0\n",
        "    X_test_orgs = []\n",
        "    np.random.seed(random_state)\n",
        "    while total_count < sample_size:\n",
        "        org = np.random.choice(single_entities, 1)[0]\n",
        "        count = orgs_df[orgs_df[provider_col] == org][\"count\"].values[0]\n",
        "        # Don't allow large orgs to dominate test data, so we get good test coverage\n",
        "        if count < sample_size / 4:\n",
        "            total_count += orgs_df[orgs_df[provider_col] == org][\"count\"].values[0]\n",
        "            X_test_orgs.append(org)\n",
        "\n",
        "    X_train_orgs = list(set(all_orgs) - set(X_test_orgs))\n",
        "\n",
        "    print(f\"Train orgs: {X_train_orgs}\")\n",
        "    print(f\"Test orgs: {X_test_orgs}\")\n",
        "\n",
        "    # Display orgs_df filtered by X_test_orgs and sorted by count descending\n",
        "    print(\"These are the test orgs and their data counts ...\")\n",
        "    display(\n",
        "        orgs_df[orgs_df[provider_col].isin(X_test_orgs)].sort_values(\n",
        "            by=\"count\", ascending=False\n",
        "        )\n",
        "    )\n",
        "\n",
        "    # Extract column rows for datasets in X_train_datasets\n",
        "    X_train = column_data[column_data[provider_col].isin(X_train_orgs)]\n",
        "\n",
        "    # Extract column rows for datasets in X_test_datasets\n",
        "    X_test = column_data[~column_data[provider_col].isin(X_train_orgs)]\n",
        "\n",
        "    train_orgs = X_train[provider_col].unique()\n",
        "    train_orgs.sort()\n",
        "    test_orgs = X_test[provider_col].unique()\n",
        "    test_orgs.sort()\n",
        "\n",
        "    print(f\"\\nTrain orgs: {train_orgs}\")\n",
        "    print(f\"\\nTest orgs: {test_orgs}\")\n",
        "\n",
        "    print(f\"\\nTrain column data: {X_train.shape}\")\n",
        "    print(f\"Test column data: {X_test.shape}\")\n",
        "\n",
        "    return X_train, X_test\n",
        "\n",
        "\n",
        "X_train, X_test = split_data(data, \"Data provider\", test_size=0.2, random_state=42)\n",
        "\n",
        "# Add a column in 'data' called test_train to indicate if row is test or train\n",
        "data[\"test_train\"] = \"train\"\n",
        "data.loc[data[\"HDX resource id\"].isin(X_test[\"HDX resource id\"]), \"test_train\"] = \"test\"\n",
        "\n",
        "# Save data\n",
        "data.to_csv(f\"{LOCAL_DATA_DIR}/hxl_hash_resources_data.csv\", index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OEPubksEAiz1"
      },
      "source": [
        "### Create LLM fine-tuning prompt files\n",
        "\n",
        "Now we have train/test datasets, we will use them to create two JSONL prompt files that can be used for fine-tuning. In my original article we opted for using a completion model, but with the release of GPT-4o we will instead generate prompts appropriate for fine-tuning a chat model (see here for more information about the available models).\n",
        "\n",
        "Each prompt is something like this …\n",
        "\n",
        "```\n",
        "{\n",
        "  \"messages\": [\n",
        "    {\n",
        "      \"role\": \"system\",\n",
        "      \"content\": \"<SYSTEM PROMPT>\"\n",
        "    },\n",
        "    {\n",
        "      \"role\": \"user\",\n",
        "      \"content\": \"<INPUT PROMPT>\"\n",
        "    },\n",
        "    {\n",
        "      \"role\": \"assistant\",\n",
        "      \"content\": \"<EXPECTED OUTPUT>\"\n",
        "    }\n",
        "  ]\n",
        "}\n",
        "\n",
        "```\n",
        "\n",
        "Note: The above has been formatted for clarity, but JSONL will have everything in one line of the prompt file.\n",
        "\n",
        "Using the data excerpts, table description, column name we collated, we can now generate prompts which look like this …"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rpzc5oAPAiz1",
        "outputId": "a35023b4-daf5-4657-929a-8d252dcb034e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Saving 2883 prompts to /content/drive/MyDrive/Colab/hxl-metadata-prediction/data//hxl_chat_prompts_train.jsonl ...\n",
            "Prompts written to /content/drive/MyDrive/Colab/hxl-metadata-prediction/data//hxl_chat_prompts_train.jsonl\n",
            "\n",
            "\n",
            "Saving 458 prompts to /content/drive/MyDrive/Colab/hxl-metadata-prediction/data//hxl_chat_prompts_test.jsonl ...\n",
            "Prompts written to /content/drive/MyDrive/Colab/hxl-metadata-prediction/data//hxl_chat_prompts_test.jsonl\n"
          ]
        }
      ],
      "source": [
        "def create_prompt_file(X_train, prompt_col, filename):\n",
        "    \"\"\"\n",
        "    Create a prompt file from a DataFrame.\n",
        "\n",
        "    Args:\n",
        "        X_train (pd.DataFrame): The DataFrame containing the prompts.\n",
        "        prompt_col (str): The name of the column containing the prompts.\n",
        "        filename (str): The name of the file to write the prompts to.\n",
        "    \"\"\"\n",
        "\n",
        "    with open(filename, \"w\") as f:\n",
        "        for index, row in X_train.iterrows():\n",
        "            f.write(row[prompt_col] + \"\\n\")\n",
        "\n",
        "    print(f\"Prompts written to {filename}\")\n",
        "\n",
        "\n",
        "def generate_chat_prompt(\n",
        "    dataset_name,\n",
        "    resource_name,\n",
        "    column_name,\n",
        "    excerpt,\n",
        "    hxl_tag=None,\n",
        "    dataset_description=\"\",\n",
        "    add_response=True,\n",
        "):\n",
        "    \"\"\"\n",
        "    Generate a chat (eg for GPT-3.5-Turbo) fine tuning prompt for HXL tags given dataset, resource, column information.\n",
        "\n",
        "    Parameters:\n",
        "    - dataset_name (str): Name of the dataset.\n",
        "    - resource_name (str): Name of the resource.\n",
        "    - column_name (str): Name of the column.\n",
        "    - excerpt (str): Examples or excerpt of the column.\n",
        "    - hxl_tag (str, optional): HXL tags for the column. Default is None.\n",
        "    - dataset_description (str, optional): Description of the dataset. Default is an empty string.\n",
        "    - add_response (bool, optional): Whether to include the response in the prompt. Default is True.\n",
        "\n",
        "    Returns:\n",
        "    - dict: A dictionary containing the prompt and optional completion/response.\n",
        "    \"\"\"\n",
        "\n",
        "    system_message = \"\"\"\n",
        "        You are an assistant that replies with HXL tags and attributes\"\n",
        "    \"\"\"\n",
        "\n",
        "    resource_name = resource_name.replace(f\"{LOCAL_DATA_DIR}/\", \"\")\n",
        "\n",
        "    column_details = (\n",
        "        f\"resource_name='{resource_name}'; \"\n",
        "        + f\"dataset_description='{dataset_description}'; \"\n",
        "        + f\"column_name:'{column_name}'; examples: {excerpt}\"\n",
        "    )\n",
        "\n",
        "    user_prompt = f\"What are the HXL tags and attributes for a column with these details? {column_details}\"\n",
        "\n",
        "    prompt = {\n",
        "        \"messages\": [\n",
        "            {\"role\": \"system\", \"content\": system_message},\n",
        "            {\"role\": \"user\", \"content\": user_prompt},\n",
        "        ]\n",
        "    }\n",
        "\n",
        "    if add_response:\n",
        "        prompt[\"messages\"].append({\"role\": \"assistant\", \"content\": hxl_tag})\n",
        "\n",
        "    # prompt = json.dumps(prompt, index=4)\n",
        "\n",
        "    return prompt\n",
        "\n",
        "\n",
        "def generate_prompts(\n",
        "    df,\n",
        "    heading_col=\"Text header\",\n",
        "    resource_name_col=\"File\",\n",
        "    tag_col=\"Hashtag with Attributes\",\n",
        "    excerpt_col=\"Data excerpt\",\n",
        "    datset_description_col=\"Data description\",\n",
        "    hxl_tag=False,\n",
        "):\n",
        "    \"\"\"\n",
        "    Generate a set of prompts for HXL tags from a DataFrame.\n",
        "\n",
        "    Parameters:\n",
        "    - df (DataFrame): Input DataFrame containing dataset, resource, column information.\n",
        "    - hxl_tag (bool, optional): Whether to include HXL tags in the prompts. Default is False.\n",
        "    - heading_col (str, optional): Name of the column containing column headers. Default is 'Text header'.\n",
        "    - resource_name_col (str, optional): Name of the column containing resource names. Default is 'File'.\n",
        "    - tag_col (str, optional): Name of the column containing HXL tags. Default is 'Hashtag with Attributes'.\n",
        "    - excerpt_col (str, optional): Name of the column containing column data excerpts. Default is 'Data excerpt'.\n",
        "    - dataset_secription_col (str, optional): Name of the column containing dataset descriptions. Default is 'Data description'.\n",
        "\n",
        "    Returns:\n",
        "    - str: A string containing JSON-formatted prompts.\n",
        "    \"\"\"\n",
        "\n",
        "    prompts = []\n",
        "    for index, row in df.iterrows():\n",
        "        if row[\"Data description\"] is None:\n",
        "            print(\"No dataset description, skipping ...\")\n",
        "            continue\n",
        "\n",
        "        prompt = generate_chat_prompt(\n",
        "            \"\",  # Dataset name\n",
        "            row[resource_name_col],\n",
        "            row[heading_col],\n",
        "            row[excerpt_col],\n",
        "            hxl_tag=row[tag_col],\n",
        "            dataset_description=row[datset_description_col],\n",
        "            add_response=hxl_tag,\n",
        "        )\n",
        "\n",
        "        prompts.append(prompt)\n",
        "\n",
        "    return prompts\n",
        "\n",
        "\n",
        "def save_all_prompts(data, dataset_type):\n",
        "    \"\"\"\n",
        "    Save all prompts generated from the given data to a JSONL file.\n",
        "\n",
        "    Args:\n",
        "        data (list): The input data to generate prompts from.\n",
        "        dataset_type (str): The type of dataset being processed.\n",
        "\n",
        "    Returns:\n",
        "        None\n",
        "    \"\"\"\n",
        "    prompt_file = f\"{LOCAL_DATA_DIR}/hxl_chat_prompts_{dataset_type}.jsonl\"\n",
        "    prompts = generate_prompts(data, hxl_tag=True)\n",
        "\n",
        "    print(f\"\\n\\nSaving {len(prompts)} prompts to {prompt_file} ...\")\n",
        "\n",
        "    with open(prompt_file, \"w\") as f:\n",
        "        for prompt in prompts:\n",
        "            f.write(json.dumps(prompt) + \"\\n\")\n",
        "\n",
        "    print(f\"Prompts written to {prompt_file}\")\n",
        "\n",
        "\n",
        "save_all_prompts(X_train, \"train\")\n",
        "save_all_prompts(X_test, \"test\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uv8B5XJRj1VV",
        "outputId": "d23b7677-ded2-482b-d81c-6d8f4d33030a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Excerpt from train prompt file:\n",
            "\n",
            "\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': 'What are the HXL tags and attributes for a column with these details? resource_name=\\'/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\\'; dataset_description=\\'The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Democratic Republic of Congo. The data includes the total number of IDP households, individual IDPs, male and female IDPs, and returnees. For instance, there are 319,283 IDP households, 1,548,732 individual IDPs, 646,805 male IDPs, 901,927 female IDPs, and 587,705 returnees.\\'; column_name:\\'Total IDP HH\\'; examples: [319283]'}, {'role': 'assistant', 'content': '#affected+hh'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': 'What are the HXL tags and attributes for a column with these details? resource_name=\\'/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\\'; dataset_description=\\'The dataset from the file \"DRC - Baseline Assessment - M23 Crisis 13 - February 20248.xlsx\" contains information on the total number of internally displaced persons (IDPs) and returnees in the Democratic Republic of Congo. The data includes the total number of IDP households, individual IDPs, male and female IDPs, and returnees. For instance, there are 319,283 IDP households, 1,548,732 individual IDPs, 646,805 male IDPs, 901,927 female IDPs, and 587,705 returnees.\\'; column_name:\\'Total Returnees\\'; examples: [587705]'}, {'role': 'assistant', 'content': '#affected+ind+returnees'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv'; dataset_description='The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.'; column_name:'country_name'; examples: ['Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan', 'Afghanistan']\"}, {'role': 'assistant', 'content': '#country'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv'; dataset_description='The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.'; column_name:'latitude'; examples: ['34.5527', '34.9568', '34.9619', '34.3033', '34.0121', '34.2743', '34.7693', '35.4474', '35.8025', '34.8046', '33.3211']\"}, {'role': 'assistant', 'content': '#geo+lat'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv'; dataset_description='The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.'; column_name:'longitude'; examples: ['69.3376', '69.6258', '68.887', '68.2174', '69.1631', '70.4529', '70.1638', '69.798', '68.9114', '67.2373', '67.812']\"}, {'role': 'assistant', 'content': '#geo+lon'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv'; dataset_description='The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.'; column_name:'indicator'; examples: ['earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake', 'earthquake']\"}, {'role': 'assistant', 'content': '#indicator+name'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/admin1-summaries-earthquake8.csv'; dataset_description='The dataset contains earthquake data for various administrative regions within Afghanistan, including information such as country name, admin1 name, latitude, longitude, aggregation method, indicator name, and indicator value. Each row represents a different administrative region with corresponding earthquake magnitude values. The dataset appears to be structured with columns for specific data attributes and rows for individual administrative regions, providing a comprehensive overview of earthquake magnitudes across different areas in Afghanistan.'; column_name:'value'; examples: ['7.58', '7.37', '7.8', '7.86', '8.26', '7.74', '7.59', '8.58', '8.47', '9.4', '7.24']\"}, {'role': 'assistant', 'content': '#indicator+num'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/fts_requirements_funding_pse3.csv'; dataset_description='The dataset contains information related to funding requirements and funding received for activities in Palestine (PSE). It includes details such as country code, activity ID, name, start and end dates, year, funding requirements in USD, total funding received in USD, and the percentage of funding received. The data shows various activities with specified funding details, as well as unspecified activities with missing information. The dataset spans multiple years, with different activities and funding amounts recorded for each year.'; column_name:'countryCode'; examples: ['PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE', 'PSE']\"}, {'role': 'assistant', 'content': '#country+code'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/fts_requirements_funding_pse3.csv'; dataset_description='The dataset contains information related to funding requirements and funding received for activities in Palestine (PSE). It includes details such as country code, activity ID, name, start and end dates, year, funding requirements in USD, total funding received in USD, and the percentage of funding received. The data shows various activities with specified funding details, as well as unspecified activities with missing information. The dataset spans multiple years, with different activities and funding amounts recorded for each year.'; column_name:'startDate'; examples: ['', '', '', '', '2024-01-01', '', '2023-01-01', '2023-10-01', '', '2022-01-01', '']\"}, {'role': 'assistant', 'content': '#date+start'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/fts_requirements_funding_pse3.csv'; dataset_description='The dataset contains information related to funding requirements and funding received for activities in Palestine (PSE). It includes details such as country code, activity ID, name, start and end dates, year, funding requirements in USD, total funding received in USD, and the percentage of funding received. The data shows various activities with specified funding details, as well as unspecified activities with missing information. The dataset spans multiple years, with different activities and funding amounts recorded for each year.'; column_name:'endDate'; examples: ['', '', '', '', '2024-12-31', '', '2023-09-30', '2023-12-31', '', '2022-12-31', '']\"}, {'role': 'assistant', 'content': '#date+end'}]}\n",
            "\n",
            "\n",
            "Excerpt from test prompt file:\n",
            "\n",
            "\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'aid'; examples: ['18401', '17770', '17721', '16858', '15268', '15113', '14826', '14230', '12788', '9286', '8561']\"}, {'role': 'assistant', 'content': '#meta+id'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'name'; examples: ['South Sudan - Floods', 'South Sudan - Population Movement from Sudan', 'South Sudan - Cholera Outbreak Upper Nile', 'South Sudan - Anticipatory Actions EVD Outbreak', 'South Sudan - Floods', 'South Sudan - Floods', 'South Sudan - Ebola Virus Disease Preparedness', 'South Sudan - Complex Emergency', 'South Sudan - Cholera', 'South Sudan - Cholera', 'Republic of South Sudan - Complex Emergency']\"}, {'role': 'assistant', 'content': '#crisis+name'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'dtype.id'; examples: ['12', '5', '1', '1', '12', '12', '1', '6', '1', '1', '7']\"}, {'role': 'assistant', 'content': '#cause+id'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'dtype.name'; examples: ['Flood', 'Population Movement', 'Epidemic', 'Epidemic', 'Flood', 'Flood', 'Epidemic', 'Complex Emergency', 'Epidemic', 'Epidemic', 'Civil Unrest']\"}, {'role': 'assistant', 'content': '#cause+name'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'status'; examples: ['0', '0', '1', '1', '1', '1', '1', '1', '1', '1', '1']\"}, {'role': 'assistant', 'content': '#status+code'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'status_display'; examples: ['Active', 'Active', 'Closed', 'Closed', 'Closed', 'Closed', 'Closed', 'Closed', 'Closed', 'Closed', 'Closed']\"}, {'role': 'assistant', 'content': '#status+name'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'start_date'; examples: ['2024-07-25T00:00:00Z', '2023-05-08T00:00:00Z', '2023-03-20T00:00:00Z', '2022-10-13T00:00:00Z', '2019-11-25T00:00:00Z', '2019-06-22T00:00:00Z', '2018-10-16T00:00:00Z', '2017-07-08T00:00:00Z', '2016-07-29T00:00:00Z', '2015-07-06T00:00:00Z', '2014-01-09T00:00:00Z']\"}, {'role': 'assistant', 'content': '#date+start'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'end_date'; examples: ['2024-09-30T00:00:00Z', '2023-08-31T00:00:00Z', '2023-07-31T00:00:00Z', '2023-01-31T00:00:00Z', '2021-07-31T00:00:00Z', '2019-11-22T00:00:00Z', '2019-02-28T00:00:00Z', '2018-12-31T00:00:00Z', '2016-12-27T00:00:00Z', '2015-10-20T00:00:00Z', '2015-07-31T00:00:00Z']\"}, {'role': 'assistant', 'content': '#date+end'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'country.iso3'; examples: ['SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD', 'SSD']\"}, {'role': 'assistant', 'content': '#country+code'}]}\n",
            "{'messages': [{'role': 'system', 'content': '\\n        You are an assistant that replies with HXL tags and attributes\"\\n    '}, {'role': 'user', 'content': \"What are the HXL tags and attributes for a column with these details? resource_name='/content/drive/MyDrive/Colab/hxl-metadata-prediction/data/IFRC Appeals Data for South Sudan8.csv'; dataset_description='The dataset contains information on various appeals and events related to South Sudan, including details such as the type of appeal, status, sector, amount requested and funded, start and end dates, as well as country-specific information like country code, region, and average household size. The data includes appeals for different crises such as floods, population movements, cholera outbreaks, and Ebola preparedness, with details on beneficiaries and confirmation needs. The dataset also includes metadata such as IDs, names, and translation modules for countries and regions.'; column_name:'country.name'; examples: ['South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan', 'South Sudan']\"}, {'role': 'assistant', 'content': '#country+name'}]}\n"
          ]
        }
      ],
      "source": [
        "for dataset_type in [\"train\", \"test\"]:\n",
        "    print(f\"\\n\\nExcerpt from {dataset_type} prompt file:\\n\\n\")\n",
        "    prompt_file = f\"{LOCAL_DATA_DIR}/hxl_chat_prompts_{dataset_type}.jsonl\"\n",
        "    with open(prompt_file) as f:\n",
        "        prompts = [json.loads(line) for line in f]\n",
        "        for p in prompts[0:10]:\n",
        "            print(p)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "hxl-prediction",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}