EHR_cleaning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d48b8474"
      },
      "source": [
        "# Part 1. Cleaning Patient Lab Ranges - David Dangond\n"
      ],
      "id": "d48b8474"
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b74fa8b1",
        "outputId": "1156e7ee-5311-4763-eb0f-529a60fa7d4a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "drive.mount('/content/drive')\n",
        "filepath = \"/content/drive/MyDrive/Spoke/\""
      ],
      "id": "b74fa8b1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0cef1f69"
      },
      "outputs": [],
      "source": [
        "not_clean = pd.read_csv(filepath + 'OCT20_range_w_demo_and_vals.tsv', sep='\\t')"
      ],
      "id": "0cef1f69"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8bb1c6b8",
        "outputId": "aff448a2-5eed-4379-f0d4-521a95528743"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-17-74593fd8aaa6>:2: DtypeWarning: Columns (6) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  res = pd.read_table(filepath + 'matched_lab_df.tsv')\n"
          ]
        }
      ],
      "source": [
        "# FOR MS RESULTS\n",
        "res = pd.read_table(filepath + 'matched_lab_df.tsv')\n"
      ],
      "id": "8bb1c6b8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "1394d243",
        "outputId": "fda30b75-2352-4533-d8da-d474056068bc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3e7bdc78-675d-434e-9464-cc4e90103cd4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>measurement_concept_id</th>\n",
              "      <th>value_as_number</th>\n",
              "      <th>value_as_concept_id</th>\n",
              "      <th>unit_concept_id</th>\n",
              "      <th>measurement_source_value</th>\n",
              "      <th>unit_source_value</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender_source_value</th>\n",
              "      <th>ethnicity_source_value</th>\n",
              "      <th>race_source_value</th>\n",
              "      <th>age_at_test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3010421</td>\n",
              "      <td>7.34</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>NaN</td>\n",
              "      <td>7.35</td>\n",
              "      <td>7.45</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>69.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8554.0</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>72.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8554.0</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>28.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>31.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>1.30</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8840.0</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dL</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>12.60</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>0.41</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>3.35</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>0.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14244786 rows × 12 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3e7bdc78-675d-434e-9464-cc4e90103cd4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3e7bdc78-675d-434e-9464-cc4e90103cd4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3e7bdc78-675d-434e-9464-cc4e90103cd4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          measurement_concept_id  value_as_number  value_as_concept_id  \\\n",
              "0                        3010421             7.34                  NaN   \n",
              "1                        3013502            69.00                  NaN   \n",
              "2                        3013502            72.00                  NaN   \n",
              "3                        3027315            28.00                  NaN   \n",
              "4                        3027315            31.00                  NaN   \n",
              "...                          ...              ...                  ...   \n",
              "14244781                 3016723             1.30                  NaN   \n",
              "14244782                 3000905            12.60                  NaN   \n",
              "14244783                 3046553             0.41                  NaN   \n",
              "14244784                 3004327             3.35                  NaN   \n",
              "14244785                 3023996             0.00                  NaN   \n",
              "\n",
              "          unit_concept_id measurement_source_value unit_source_value  \\\n",
              "0                     NaN                  LAB5808               NaN   \n",
              "1                  8554.0                  LAB5808                 %   \n",
              "2                  8554.0                  LAB5809                 %   \n",
              "3                     NaN                  LAB5808             mm Hg   \n",
              "4                     NaN                  LAB5809             mm Hg   \n",
              "...                   ...                      ...               ...   \n",
              "14244781           8840.0                  LAB3001             mg/dL   \n",
              "14244782              NaN                   LAB294           x10E9/L   \n",
              "14244783              NaN                   LAB293           x10E9/L   \n",
              "14244784              NaN                   LAB293           x10E9/L   \n",
              "14244785              NaN                   LC3743    mmol/mol creat   \n",
              "\n",
              "          range_low  range_high gender_source_value  ethnicity_source_value  \\\n",
              "0              7.35        7.45              Female  Not Hispanic or Latino   \n",
              "1             95.00       99.00              Female  Not Hispanic or Latino   \n",
              "2             95.00       99.00              Female  Not Hispanic or Latino   \n",
              "3             80.00      100.00              Female  Not Hispanic or Latino   \n",
              "4             80.00      100.00              Female  Not Hispanic or Latino   \n",
              "...             ...         ...                 ...                     ...   \n",
              "14244781       0.61        1.24                Male      Hispanic or Latino   \n",
              "14244782       3.40       10.00              Female                 Unknown   \n",
              "14244783       0.00        0.20              Female      Hispanic or Latino   \n",
              "14244784       0.90        2.90                Male  Not Hispanic or Latino   \n",
              "14244785       1.00       14.00                Male      Hispanic or Latino   \n",
              "\n",
              "           race_source_value  age_at_test  \n",
              "0                      Asian            0  \n",
              "1                      Asian            0  \n",
              "2                      Asian            0  \n",
              "3                      Asian            0  \n",
              "4                      Asian            0  \n",
              "...                      ...          ...  \n",
              "14244781  White or Caucasian           53  \n",
              "14244782             Unknown           71  \n",
              "14244783  White or Caucasian            2  \n",
              "14244784  White or Caucasian           36  \n",
              "14244785             Unknown           13  \n",
              "\n",
              "[14244786 rows x 12 columns]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "not_clean"
      ],
      "id": "1394d243"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "05d74398",
        "outputId": "12f44e5a-1ba4-4bd0-d3cf-bdcd7e9330c1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1276"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(set(res[~res['OMOPid'].isin(list(set(not_clean['measurement_concept_id'])))]['OMOPid']))"
      ],
      "id": "05d74398"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "068e0f3b",
        "outputId": "62c7a577-e574-46bb-d875-c43c9fa9ded4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1941"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(set(res['OMOPid']))"
      ],
      "id": "068e0f3b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f38559bc",
        "outputId": "f7413c19-aa6f-4521-9362-a7387589404e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "983"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(set(not_clean['measurement_concept_id']))"
      ],
      "id": "f38559bc"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a2d3f646",
        "outputId": "fb87377b-a32c-47c8-9214-14fb30f5c0a7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "number of labs from original data set included in MS Dataset:  665\n",
            "number of labs from original data set not included in MS Dataset:  318\n"
          ]
        }
      ],
      "source": [
        "# ms = not_clean[not_clean['measurement_concept_id'].isin(list(set(res['OMOPid'])))]\n",
        "ms = res[res['OMOPid'].isin(list(set(not_clean['measurement_concept_id'])))]\n",
        "\n",
        "print('number of labs from original data set included in MS Dataset: ', len(set(ms['OMOPid'])))\n",
        "\n",
        "\n",
        "not_ms = not_clean[~not_clean['measurement_concept_id'].isin(list(set(res['OMOPid'])))]\n",
        "\n",
        "print('number of labs from original data set not included in MS Dataset: ', len(set(not_ms['measurement_concept_id'])))"
      ],
      "id": "a2d3f646"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "572d8788",
        "outputId": "ac1db38d-9653-47f8-9bb2-3a503e83b504"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-550f0904-ef81-4c03-afae-9483f231ae23\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>measurement_concept_id</th>\n",
              "      <th>value_as_number</th>\n",
              "      <th>value_as_concept_id</th>\n",
              "      <th>unit_concept_id</th>\n",
              "      <th>measurement_source_value</th>\n",
              "      <th>unit_source_value</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender_source_value</th>\n",
              "      <th>ethnicity_source_value</th>\n",
              "      <th>race_source_value</th>\n",
              "      <th>age_at_test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3010421</td>\n",
              "      <td>7.34</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>NaN</td>\n",
              "      <td>7.35</td>\n",
              "      <td>7.45</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>69.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8554.0</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>72.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8554.0</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>28.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>31.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>1.30</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8840.0</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dL</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>12.60</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>0.41</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>3.35</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>0.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14244786 rows × 12 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-550f0904-ef81-4c03-afae-9483f231ae23')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-550f0904-ef81-4c03-afae-9483f231ae23 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-550f0904-ef81-4c03-afae-9483f231ae23');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          measurement_concept_id  value_as_number  value_as_concept_id  \\\n",
              "0                        3010421             7.34                  NaN   \n",
              "1                        3013502            69.00                  NaN   \n",
              "2                        3013502            72.00                  NaN   \n",
              "3                        3027315            28.00                  NaN   \n",
              "4                        3027315            31.00                  NaN   \n",
              "...                          ...              ...                  ...   \n",
              "14244781                 3016723             1.30                  NaN   \n",
              "14244782                 3000905            12.60                  NaN   \n",
              "14244783                 3046553             0.41                  NaN   \n",
              "14244784                 3004327             3.35                  NaN   \n",
              "14244785                 3023996             0.00                  NaN   \n",
              "\n",
              "          unit_concept_id measurement_source_value unit_source_value  \\\n",
              "0                     NaN                  LAB5808               NaN   \n",
              "1                  8554.0                  LAB5808                 %   \n",
              "2                  8554.0                  LAB5809                 %   \n",
              "3                     NaN                  LAB5808             mm Hg   \n",
              "4                     NaN                  LAB5809             mm Hg   \n",
              "...                   ...                      ...               ...   \n",
              "14244781           8840.0                  LAB3001             mg/dL   \n",
              "14244782              NaN                   LAB294           x10E9/L   \n",
              "14244783              NaN                   LAB293           x10E9/L   \n",
              "14244784              NaN                   LAB293           x10E9/L   \n",
              "14244785              NaN                   LC3743    mmol/mol creat   \n",
              "\n",
              "          range_low  range_high gender_source_value  ethnicity_source_value  \\\n",
              "0              7.35        7.45              Female  Not Hispanic or Latino   \n",
              "1             95.00       99.00              Female  Not Hispanic or Latino   \n",
              "2             95.00       99.00              Female  Not Hispanic or Latino   \n",
              "3             80.00      100.00              Female  Not Hispanic or Latino   \n",
              "4             80.00      100.00              Female  Not Hispanic or Latino   \n",
              "...             ...         ...                 ...                     ...   \n",
              "14244781       0.61        1.24                Male      Hispanic or Latino   \n",
              "14244782       3.40       10.00              Female                 Unknown   \n",
              "14244783       0.00        0.20              Female      Hispanic or Latino   \n",
              "14244784       0.90        2.90                Male  Not Hispanic or Latino   \n",
              "14244785       1.00       14.00                Male      Hispanic or Latino   \n",
              "\n",
              "           race_source_value  age_at_test  \n",
              "0                      Asian            0  \n",
              "1                      Asian            0  \n",
              "2                      Asian            0  \n",
              "3                      Asian            0  \n",
              "4                      Asian            0  \n",
              "...                      ...          ...  \n",
              "14244781  White or Caucasian           53  \n",
              "14244782             Unknown           71  \n",
              "14244783  White or Caucasian            2  \n",
              "14244784  White or Caucasian           36  \n",
              "14244785             Unknown           13  \n",
              "\n",
              "[14244786 rows x 12 columns]"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "not_clean"
      ],
      "id": "572d8788"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "d5100c51",
        "outputId": "6d5b6b0b-f6c1-427b-e118-cd2b8fcf1b1c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-a5c09887-f5b0-4983-b689-dca25ff7a3dc\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cohort_id</th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>omop_date</th>\n",
              "      <th>value_as_number</th>\n",
              "      <th>value_as_concept_id</th>\n",
              "      <th>unit_concept_id</th>\n",
              "      <th>unit_source_value</th>\n",
              "      <th>measurement_source_value</th>\n",
              "      <th>value_source_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>3013721</td>\n",
              "      <td>2009-09-03</td>\n",
              "      <td>18.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8645.0</td>\n",
              "      <td>U/L</td>\n",
              "      <td>1526000</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>3019550</td>\n",
              "      <td>2009-02-10</td>\n",
              "      <td>138.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8753.0</td>\n",
              "      <td>MMOL/L</td>\n",
              "      <td>1534098</td>\n",
              "      <td>138</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>3035995</td>\n",
              "      <td>2009-09-03</td>\n",
              "      <td>59.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8645.0</td>\n",
              "      <td>U/L</td>\n",
              "      <td>1525848</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>3044491</td>\n",
              "      <td>2009-02-10</td>\n",
              "      <td>122.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8840.0</td>\n",
              "      <td>MG/DL</td>\n",
              "      <td>676</td>\n",
              "      <td>122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>3022192</td>\n",
              "      <td>2009-02-10</td>\n",
              "      <td>50.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8840.0</td>\n",
              "      <td>MG/DL</td>\n",
              "      <td>1552156</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1294817</th>\n",
              "      <td>41177</td>\n",
              "      <td>3000905</td>\n",
              "      <td>1998-02-26</td>\n",
              "      <td>2.10</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9444.0</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>1004</td>\n",
              "      <td>2.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1294818</th>\n",
              "      <td>41177</td>\n",
              "      <td>3015884</td>\n",
              "      <td>1998-08-14</td>\n",
              "      <td>227.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8837.0</td>\n",
              "      <td>ug/dL</td>\n",
              "      <td>1900562</td>\n",
              "      <td>227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1294819</th>\n",
              "      <td>41177</td>\n",
              "      <td>3004501</td>\n",
              "      <td>1998-08-14</td>\n",
              "      <td>88.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8840.0</td>\n",
              "      <td>mg/dL</td>\n",
              "      <td>1510655</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1294820</th>\n",
              "      <td>41177</td>\n",
              "      <td>3023314</td>\n",
              "      <td>1998-02-26</td>\n",
              "      <td>43.70</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8554.0</td>\n",
              "      <td>PERCENT</td>\n",
              "      <td>1534436</td>\n",
              "      <td>43.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1295022</th>\n",
              "      <td>41241</td>\n",
              "      <td>3009201</td>\n",
              "      <td>2018-05-17</td>\n",
              "      <td>0.54</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9040.0</td>\n",
              "      <td>mIU/L</td>\n",
              "      <td>2465</td>\n",
              "      <td>0.54</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>660445 rows × 9 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a5c09887-f5b0-4983-b689-dca25ff7a3dc')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a5c09887-f5b0-4983-b689-dca25ff7a3dc button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a5c09887-f5b0-4983-b689-dca25ff7a3dc');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         cohort_id   OMOPid   omop_date  value_as_number  value_as_concept_id  \\\n",
              "0                0  3013721  2009-09-03            18.00                  NaN   \n",
              "1                0  3019550  2009-02-10           138.00                  NaN   \n",
              "2                0  3035995  2009-09-03            59.00                  NaN   \n",
              "3                0  3044491  2009-02-10           122.00                  NaN   \n",
              "4                0  3022192  2009-02-10            50.00                  NaN   \n",
              "...            ...      ...         ...              ...                  ...   \n",
              "1294817      41177  3000905  1998-02-26             2.10                  NaN   \n",
              "1294818      41177  3015884  1998-08-14           227.00                  NaN   \n",
              "1294819      41177  3004501  1998-08-14            88.00                  NaN   \n",
              "1294820      41177  3023314  1998-02-26            43.70                  NaN   \n",
              "1295022      41241  3009201  2018-05-17             0.54                  NaN   \n",
              "\n",
              "         unit_concept_id unit_source_value  measurement_source_value  \\\n",
              "0                 8645.0               U/L                   1526000   \n",
              "1                 8753.0            MMOL/L                   1534098   \n",
              "2                 8645.0               U/L                   1525848   \n",
              "3                 8840.0             MG/DL                       676   \n",
              "4                 8840.0             MG/DL                   1552156   \n",
              "...                  ...               ...                       ...   \n",
              "1294817           9444.0           x10E9/L                      1004   \n",
              "1294818           8837.0             ug/dL                   1900562   \n",
              "1294819           8840.0             mg/dL                   1510655   \n",
              "1294820           8554.0           PERCENT                   1534436   \n",
              "1295022           9040.0             mIU/L                      2465   \n",
              "\n",
              "        value_source_value  \n",
              "0                       18  \n",
              "1                      138  \n",
              "2                       59  \n",
              "3                      122  \n",
              "4                       50  \n",
              "...                    ...  \n",
              "1294817                2.1  \n",
              "1294818                227  \n",
              "1294819                 88  \n",
              "1294820               43.7  \n",
              "1295022               0.54  \n",
              "\n",
              "[660445 rows x 9 columns]"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ms"
      ],
      "id": "d5100c51"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3ee12d5c"
      },
      "outputs": [],
      "source": [
        "renamed = not_clean.rename(columns = {'measurement_concept_id': 'OMOPid', 'measurement_source_value' : 'manufacturer', 'unit_source_value':'unit', 'gender_source_value':'gender','ethnicity_source_value':'ethnicity','race_source_value':'race','age_at_test':'age'})\n",
        "filtered = renamed[['OMOPid', 'manufacturer', 'unit', 'range_low', 'range_high', 'gender', 'ethnicity', 'race', 'age']]\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "id": "3ee12d5c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9a55277b"
      },
      "outputs": [],
      "source": [
        "omc_raw = pd.read_csv(filepath + 'old_measurement_concepts.tsv', sep='\\t')\n",
        "omc = omc_raw.rename(columns={'concept_id': \"OMOPid\", 'concept_code': 'LOINCid'})"
      ],
      "id": "9a55277b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "214babc7",
        "outputId": "49a3e624-ef19-40a0-a027-539601d2a4af"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "old_measurement_concepts:\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ea8cf9fb-c654-4dfb-a6da-054bd544179d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>concept_name</th>\n",
              "      <th>domain_id</th>\n",
              "      <th>vocabulary_id</th>\n",
              "      <th>concept_class_id</th>\n",
              "      <th>standard_concept</th>\n",
              "      <th>LOINCid</th>\n",
              "      <th>valid_start_date</th>\n",
              "      <th>valid_end_date</th>\n",
              "      <th>invalid_reason</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>44786677</td>\n",
              "      <td>Genetic risk algorithm sensitivity [Ratio]</td>\n",
              "      <td>Measurement</td>\n",
              "      <td>LOINC</td>\n",
              "      <td>Clinical Observation</td>\n",
              "      <td>S</td>\n",
              "      <td>74020-9</td>\n",
              "      <td>2013-07-23</td>\n",
              "      <td>2099-12-31</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>44786872</td>\n",
              "      <td>Fondaparinux [Mass/volume] in Platelet poor pl...</td>\n",
              "      <td>Measurement</td>\n",
              "      <td>LOINC</td>\n",
              "      <td>Lab Test</td>\n",
              "      <td>S</td>\n",
              "      <td>74217-1</td>\n",
              "      <td>2013-10-28</td>\n",
              "      <td>2099-12-31</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>44786663</td>\n",
              "      <td>Serotonin release 0.5 IU/mL heparin.unfraction...</td>\n",
              "      <td>Measurement</td>\n",
              "      <td>LOINC</td>\n",
              "      <td>Lab Test</td>\n",
              "      <td>S</td>\n",
              "      <td>74005-0</td>\n",
              "      <td>2013-07-22</td>\n",
              "      <td>2099-12-31</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>44786754</td>\n",
              "      <td>Tacrolimus [Mass/volume] in Blood by LC/MS/MS</td>\n",
              "      <td>Measurement</td>\n",
              "      <td>LOINC</td>\n",
              "      <td>Lab Test</td>\n",
              "      <td>S</td>\n",
              "      <td>74097-7</td>\n",
              "      <td>2013-08-29</td>\n",
              "      <td>2099-12-31</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>42528717</td>\n",
              "      <td>Cashew nut recombinant (rAna o) 3 IgE Ab [Unit...</td>\n",
              "      <td>Measurement</td>\n",
              "      <td>LOINC</td>\n",
              "      <td>Lab Test</td>\n",
              "      <td>S</td>\n",
              "      <td>82539-8</td>\n",
              "      <td>2017-02-21</td>\n",
              "      <td>2099-12-31</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ea8cf9fb-c654-4dfb-a6da-054bd544179d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ea8cf9fb-c654-4dfb-a6da-054bd544179d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ea8cf9fb-c654-4dfb-a6da-054bd544179d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "     OMOPid                                       concept_name    domain_id  \\\n",
              "0  44786677         Genetic risk algorithm sensitivity [Ratio]  Measurement   \n",
              "1  44786872  Fondaparinux [Mass/volume] in Platelet poor pl...  Measurement   \n",
              "2  44786663  Serotonin release 0.5 IU/mL heparin.unfraction...  Measurement   \n",
              "3  44786754      Tacrolimus [Mass/volume] in Blood by LC/MS/MS  Measurement   \n",
              "4  42528717  Cashew nut recombinant (rAna o) 3 IgE Ab [Unit...  Measurement   \n",
              "\n",
              "  vocabulary_id      concept_class_id standard_concept  LOINCid  \\\n",
              "0         LOINC  Clinical Observation                S  74020-9   \n",
              "1         LOINC              Lab Test                S  74217-1   \n",
              "2         LOINC              Lab Test                S  74005-0   \n",
              "3         LOINC              Lab Test                S  74097-7   \n",
              "4         LOINC              Lab Test                S  82539-8   \n",
              "\n",
              "  valid_start_date valid_end_date invalid_reason  \n",
              "0       2013-07-23     2099-12-31            NaN  \n",
              "1       2013-10-28     2099-12-31            NaN  \n",
              "2       2013-07-22     2099-12-31            NaN  \n",
              "3       2013-08-29     2099-12-31            NaN  \n",
              "4       2017-02-21     2099-12-31            NaN  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "print('old_measurement_concepts:')\n",
        "# not useful, many labs done multiple times\n",
        "# omcUnique = omc.drop_duplicates(subset = ['OMOPid','LOINCid'])\n",
        "# print(len(omcUnique))\n",
        "# print(len(omc))\n",
        "display(omc.head())"
      ],
      "id": "214babc7"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f2a33474",
        "outputId": "f2e1f70f-42d7-48df-afcc-1d08791d4512"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{44786677: '74020-9',\n",
              " 44786872: '74217-1',\n",
              " 44786663: '74005-0',\n",
              " 44786754: '74097-7',\n",
              " 42528717: '82539-8',\n",
              " 46235951: '77741-7',\n",
              " 46236302: '77636-9',\n",
              " 43055684: '72824-6',\n",
              " 43055120: '72493-0',\n",
              " 43055151: '72524-2',\n",
              " 43055153: '72526-7',\n",
              " 43055279: '72314-8',\n",
              " 43055441: '72607-5',\n",
              " 43055122: '72495-5',\n",
              " 43055144: '72517-6',\n",
              " 43055457: '72623-2',\n",
              " 43055031: '72400-5',\n",
              " 43055560: '72726-3',\n",
              " 43055562: '72728-9',\n",
              " 43055033: '72402-1',\n",
              " 43055022: '72391-6',\n",
              " 43055121: '72494-8',\n",
              " 44786773: '74116-5',\n",
              " 21493555: '80674-5',\n",
              " 21493565: '80684-4',\n",
              " 40761514: '58413-6',\n",
              " 40761531: '58430-0',\n",
              " 40760859: '57749-4',\n",
              " 40760911: '57802-1',\n",
              " 40761010: '57901-1',\n",
              " 46237016: '77174-1',\n",
              " 46235436: '75654-4',\n",
              " 46236749: '77039-6',\n",
              " 46236971: '78002-3',\n",
              " 40760439: '57320-4',\n",
              " 46235690: '75882-1',\n",
              " 46235955: '77745-8',\n",
              " 46235190: '76651-9',\n",
              " 46236293: '77627-8',\n",
              " 40762156: '59063-8',\n",
              " 46237030: '77207-9',\n",
              " 40762529: '59466-3',\n",
              " 40765679: '62947-7',\n",
              " 40765841: '63109-3',\n",
              " 40765889: '63157-2',\n",
              " 40766283: '63557-3',\n",
              " 40771461: '68916-6',\n",
              " 40771515: '68990-1',\n",
              " 40771527: '69002-4',\n",
              " 40771573: '69052-9',\n",
              " 40762114: '59021-6',\n",
              " 40762115: '59022-4',\n",
              " 40762154: '59061-2',\n",
              " 40772044: '69549-4',\n",
              " 40761526: '58425-0',\n",
              " 40761808: '58714-7',\n",
              " 21492659: '79381-0',\n",
              " 21493559: '80678-6',\n",
              " 21493563: '80682-8',\n",
              " 21490850: '76212-0',\n",
              " 40765015: '62254-8',\n",
              " 40765068: '62321-5',\n",
              " 40765103: '62362-9',\n",
              " 21494812: '80220-7',\n",
              " 3000483: '2339-0',\n",
              " 3000699: '3528-7',\n",
              " 3000876: '6082-2',\n",
              " 3000936: '6154-9',\n",
              " 3002106: '30241-4',\n",
              " 3002135: '31107-6',\n",
              " 3002159: '25082-9',\n",
              " 3002252: '4098-0',\n",
              " 3002302: '5199-5',\n",
              " 3001571: '17817-8',\n",
              " 3001585: '13485-8',\n",
              " 3001727: '15994-7',\n",
              " 3001784: '12841-3',\n",
              " 3001832: '17522-4',\n",
              " 3000257: '15841-0',\n",
              " 3005488: '16111-7',\n",
              " 3005527: '501-7',\n",
              " 3005710: '5630-9',\n",
              " 3005804: '17790-7',\n",
              " 3002051: '30542-5',\n",
              " 3005008: '5609-3',\n",
              " 3005029: '2881-1',\n",
              " 3005190: '16249-5',\n",
              " 3005225: '14805-6',\n",
              " 3005333: '6219-0',\n",
              " 3015399: '30248-9',\n",
              " 3015546: '20639-1',\n",
              " 3015637: '26459-8',\n",
              " 3015697: '12710-0',\n",
              " 3015736: '2756-5',\n",
              " 3015753: '29515-4',\n",
              " 3015789: '906-8',\n",
              " 3005900: '17729-5',\n",
              " 3006109: '3187-2',\n",
              " 3006318: '25114-0',\n",
              " 40759913: '56794-1',\n",
              " 3006689: '3638-4',\n",
              " 3006734: '6189-5',\n",
              " 3006775: '27094-2',\n",
              " 3004691: '8251-1',\n",
              " 3004708: '27092-6',\n",
              " 3032693: '49502-8',\n",
              " 3015830: '26656-9',\n",
              " 3015902: '3894-3',\n",
              " 3016244: '20448-7',\n",
              " 3016255: '1005-8',\n",
              " 3016266: '5922-0',\n",
              " 3016273: '31209-0',\n",
              " 3016407: '3255-7',\n",
              " 3016459: '6178-8',\n",
              " 3016515: '2943-9',\n",
              " 3016617: '13358-7',\n",
              " 3016939: '17781-6',\n",
              " 3016948: '3184-9',\n",
              " 3017042: '3935-4',\n",
              " 3007321: '2765-6',\n",
              " 3007352: '2091-7',\n",
              " 3032813: '43102-3',\n",
              " 3032897: '35175-9',\n",
              " 3032984: '49898-0',\n",
              " 3032999: '49711-5',\n",
              " 3033104: '41863-2',\n",
              " 3008893: '2986-8',\n",
              " 3008901: '6271-1',\n",
              " 3008943: '30412-1',\n",
              " 3009172: '20636-7',\n",
              " 3009299: '3284-7',\n",
              " 3050380: '47528-5',\n",
              " 3050449: '49002-9',\n",
              " 3009383: '12173-1',\n",
              " 3009542: '20570-8',\n",
              " 3009582: '1533-9',\n",
              " 3009927: '2333-3',\n",
              " 3010039: '21025-2',\n",
              " 3034933: '42607-2',\n",
              " 3035020: '24442-6',\n",
              " 3035113: '20626-8',\n",
              " 3035128: '19053-8',\n",
              " 40767125: '64414-6',\n",
              " 3002278: '1854-9',\n",
              " 3002291: '6842-9',\n",
              " 3003985: '6873-4',\n",
              " 3004266: '19147-8',\n",
              " 3016767: '13735-6',\n",
              " 3006865: '241-0',\n",
              " 3006906: '17861-6',\n",
              " 3007359: '1971-1',\n",
              " 3007384: '27400-1',\n",
              " 3007490: '15875-8',\n",
              " 3007498: '30567-2',\n",
              " 3007561: '27923-2',\n",
              " 3007719: '1657-6',\n",
              " 3007723: '27851-5',\n",
              " 3008561: '13590-5',\n",
              " 3008612: '13734-9',\n",
              " 3008714: '32166-1',\n",
              " 3034523: '3536-0',\n",
              " 3034655: '42482-0',\n",
              " 3034806: '6087-1',\n",
              " 3034901: '35386-2',\n",
              " 3017575: '19146-0',\n",
              " 3017662: '25097-7',\n",
              " 3017750: '24446-7',\n",
              " 3017829: '21748-9',\n",
              " 3017886: '27392-0',\n",
              " 3017958: '6257-0',\n",
              " 3017974: '6864-3',\n",
              " 3018095: '30405-5',\n",
              " 3018214: '25098-5',\n",
              " 3002020: '3377-9',\n",
              " 3002035: '29643-4',\n",
              " 3002212: '30234-9',\n",
              " 3002248: '3978-4',\n",
              " 3004394: '7340-3',\n",
              " 3004418: '13928-7',\n",
              " 3004560: '26475-4',\n",
              " 3004593: '31032-6',\n",
              " 3004640: '11051-0',\n",
              " 3004653: '3982-6',\n",
              " 3004772: '5393-4',\n",
              " 3004775: '19153-6',\n",
              " 3004783: '26972-0',\n",
              " 3004879: '10579-1',\n",
              " 3004880: '5092-2',\n",
              " 3005148: '1695-6',\n",
              " 3005171: '5281-1',\n",
              " 3007808: '2915-7',\n",
              " 3002996: '569-4',\n",
              " 3003038: '15385-8',\n",
              " 3003159: '23860-0',\n",
              " 3005222: '1165-0',\n",
              " 3005229: '2868-8',\n",
              " 3005335: '22672-0',\n",
              " 3005440: '31104-3',\n",
              " 3014717: '9407-8',\n",
              " 3014749: '20408-1',\n",
              " 3014856: '10728-4',\n",
              " 3014929: '8236-2',\n",
              " 3014958: '21712-5',\n",
              " 3015160: '2148-5',\n",
              " 3005685: '1809-3',\n",
              " 3006471: '12723-3',\n",
              " 3006567: '2079-2',\n",
              " 3006627: '24007-7',\n",
              " 3006665: '27395-3',\n",
              " 3006762: '20656-5',\n",
              " 3006828: '32024-2',\n",
              " 3003525: '6456-8',\n",
              " 3005716: '15732-1',\n",
              " 3015682: '5225-8',\n",
              " 3015802: '6113-5',\n",
              " 3015813: '3182-3',\n",
              " 3015884: '2191-5',\n",
              " 3015973: '15929-3',\n",
              " 3006013: '6045-9',\n",
              " 3006157: '31146-4',\n",
              " 3033882: '14288-5',\n",
              " 3034043: '42664-3',\n",
              " 3008699: '15557-2',\n",
              " 3017894: '6185-3',\n",
              " 3017930: '23877-4',\n",
              " 3017980: '29965-1',\n",
              " 3018031: '9800-4',\n",
              " 3009119: '13728-1',\n",
              " 3009190: '5375-1',\n",
              " 3000456: '7791-7',\n",
              " 3000610: '15387-4',\n",
              " 3000620: '4485-9',\n",
              " 3000637: '12228-3',\n",
              " 3000661: '18919-1',\n",
              " 3000669: '23940-0',\n",
              " 3000715: '13793-5',\n",
              " 3035055: '3772-1',\n",
              " 3001706: '3834-9',\n",
              " 3001754: '18250-1',\n",
              " 3001802: '14959-1',\n",
              " 3001818: '3690-5',\n",
              " 3001881: '2432-3',\n",
              " 3002022: '27820-0',\n",
              " 3002527: '6090-5',\n",
              " 3002529: '883-9',\n",
              " 3002538: '7558-0',\n",
              " 40761804: '58710-5',\n",
              " 40761860: '58766-7',\n",
              " 40762162: '59069-5',\n",
              " 3035318: '22679-5',\n",
              " 3018625: '29507-1',\n",
              " 3018669: '6033-5',\n",
              " 3018677: '14979-9',\n",
              " 3018686: '2217-8',\n",
              " 3002751: '26019-0',\n",
              " 3002752: '21194-6',\n",
              " 3002803: '9308-8',\n",
              " 3006248: '18361-6',\n",
              " 3003142: '9766-7',\n",
              " 3005503: '16099-4',\n",
              " 3005577: '14956-7',\n",
              " 3005627: '9306-2',\n",
              " 3005673: '17856-6',\n",
              " 3005757: '3193-0',\n",
              " 3005788: '7152-2',\n",
              " 3005854: '7790-9',\n",
              " 3005941: '22636-5',\n",
              " 3005987: '5251-4',\n",
              " 3006070: '6954-2',\n",
              " 3006163: '5022-9',\n",
              " 3006173: '29525-3',\n",
              " 3006175: '8247-9',\n",
              " 3006638: '4059-2',\n",
              " 3016728: '20646-6',\n",
              " 3016742: '15859-2',\n",
              " 3016750: '13362-9',\n",
              " 3016806: '12941-1',\n",
              " 3016961: '8172-9',\n",
              " 3004187: '3793-7',\n",
              " 3004202: '18955-5',\n",
              " 3004249: '8480-6',\n",
              " 3004254: '20648-2',\n",
              " 3004291: '8134-9',\n",
              " 3014791: '1884-6',\n",
              " 3014848: '15343-7',\n",
              " 3033445: '41479-7',\n",
              " 3017526: '9345-0',\n",
              " 3017788: '11146-8',\n",
              " 3017840: '6800-7',\n",
              " 3007700: '6037-6',\n",
              " 3007822: '13438-7',\n",
              " 3007880: '15869-1',\n",
              " 3007885: '5695-2',\n",
              " 3033575: '742-7',\n",
              " 3034006: '42602-3',\n",
              " 3034108: '20778-7',\n",
              " 3034127: '24436-8',\n",
              " 3005013: '10886-0',\n",
              " 3005018: '30566-4',\n",
              " 3005168: '2501-5',\n",
              " 3005169: '567-8',\n",
              " 3005235: '6153-1',\n",
              " 3008848: '2077-6',\n",
              " 3008895: '1656-8',\n",
              " 3008976: '13283-7',\n",
              " 3009105: '20409-9',\n",
              " 3009169: '2150-1',\n",
              " 3008043: '17609-9',\n",
              " 3008193: '21003-9',\n",
              " 3009718: '9832-7',\n",
              " 3009827: '1835-8',\n",
              " 3010042: '5572-3',\n",
              " 3035212: '35303-7',\n",
              " 3035509: '35670-9',\n",
              " 3035569: '2283-0',\n",
              " 3035583: '5821-4',\n",
              " 3018204: '5229-0',\n",
              " 3018256: '25110-8',\n",
              " 3000067: '2731-8',\n",
              " 3000138: '5583-0',\n",
              " 3000185: '2502-3',\n",
              " 21493558: '80677-8',\n",
              " 3003809: '30233-1',\n",
              " 3003816: '30534-2',\n",
              " 3003891: '8100-0',\n",
              " 3004109: '21376-9',\n",
              " 3004167: '8245-3',\n",
              " 3004239: '2162-6',\n",
              " 3004308: '12382-8',\n",
              " 3004786: '24312-1',\n",
              " 3004833: '21081-5',\n",
              " 3005089: '21709-1',\n",
              " 3005136: '6075-6',\n",
              " 3005156: '30075-6',\n",
              " 3005186: '1159-3',\n",
              " 3004360: '13483-3',\n",
              " 3004624: '2116-2',\n",
              " 3004705: '18961-3',\n",
              " 3005769: '3494-2',\n",
              " 3005807: '24471-5',\n",
              " 3005828: '5732-3',\n",
              " 3006072: '18879-7',\n",
              " 3006135: '772-4',\n",
              " 3016330: '2829-0',\n",
              " 3016417: '1805-1',\n",
              " 3016436: '2532-0',\n",
              " 3016473: '18723-7',\n",
              " 3016589: '32009-3',\n",
              " 40763086: '60026-2',\n",
              " 40765014: '62253-0',\n",
              " 40765039: '62291-0',\n",
              " 40765204: '62467-6',\n",
              " 3005424: '8277-6',\n",
              " 3005456: '6298-4',\n",
              " 3005481: '802-9',\n",
              " 3005519: '13482-5',\n",
              " 3005562: '6069-9',\n",
              " 3005564: '5949-3',\n",
              " 3005578: '13626-7',\n",
              " 3015142: '6270-3',\n",
              " 3015195: '22703-3',\n",
              " 3015479: '533-0',\n",
              " 3015501: '27378-9',\n",
              " 3006687: '2308-5',\n",
              " 3006747: '25108-2',\n",
              " 3006888: '7151-4',\n",
              " 3007070: '2085-9',\n",
              " 3006209: '844-1',\n",
              " 3006262: '29910-7',\n",
              " 3006345: '3042-9',\n",
              " 3006355: '9360-9',\n",
              " 3006461: '11020-5',\n",
              " 3006517: '10587-4',\n",
              " 3006578: '10659-1',\n",
              " 3006615: '2236-8',\n",
              " 3006631: '32018-4',\n",
              " 3000030: '31099-5',\n",
              " 3000120: '21036-9',\n",
              " 3000144: '19343-3',\n",
              " 3000180: '14054-1',\n",
              " 3000194: '21108-6',\n",
              " 3000201: '19415-9',\n",
              " 3007082: '15280-1',\n",
              " 3007256: '2693-0',\n",
              " 3007456: '6194-5',\n",
              " 3007461: '26515-7',\n",
              " 3007527: '9804-6',\n",
              " 3000150: '6076-4',\n",
              " 3000307: '31562-2',\n",
              " 3000422: '23918-6',\n",
              " 3000515: '27811-9',\n",
              " 3000285: '2947-0',\n",
              " 3000426: '16027-5',\n",
              " 3000497: '22450-1',\n",
              " 3000571: '19346-6',\n",
              " 3000598: '12234-1',\n",
              " 3000685: '25835-0',\n",
              " 3000800: '21343-9',\n",
              " 3008159: '20566-6',\n",
              " 3008204: '32167-9',\n",
              " 3008295: '2692-2',\n",
              " 3008377: '20650-8',\n",
              " 3008423: '21751-3',\n",
              " 3008424: '20421-4',\n",
              " 3034388: '25092-8',\n",
              " 3034411: '2436-4',\n",
              " 3034421: '16124-0',\n",
              " 3000998: '7556-4',\n",
              " 3001101: '20401-6',\n",
              " 3018163: '19152-8',\n",
              " 3018173: '5693-7',\n",
              " 3018194: '6186-1',\n",
              " 3018229: '749-2',\n",
              " 40766812: '64096-1',\n",
              " 40766945: '64234-8',\n",
              " 3000670: '3992-5',\n",
              " 3000722: '14286-9',\n",
              " 3000899: '25296-5',\n",
              " 3034807: '7802-2',\n",
              " 3034868: '38180-6',\n",
              " 3035213: '38163-2',\n",
              " 3018471: '23878-2',\n",
              " 3018765: '6029-3',\n",
              " 3000846: '15691-9',\n",
              " 3000924: '11268-0',\n",
              " 3000984: '2568-4',\n",
              " 3001420: '19123-9',\n",
              " 3001591: '16650-4',\n",
              " 3001718: '27055-3',\n",
              " 3001804: '13629-1',\n",
              " 3001879: '20545-0',\n",
              " 3001903: '21273-8',\n",
              " 3002042: '6915-3',\n",
              " 3002060: '16042-4',\n",
              " 3002069: '15013-6',\n",
              " 3002079: '2956-1',\n",
              " 3002112: '2282-2',\n",
              " 3002187: '6234-9',\n",
              " 3002100: '9767-5',\n",
              " 3002234: '16107-5',\n",
              " 3002285: '10626-0',\n",
              " 3002313: '7335-3',\n",
              " 3003120: '30005-3',\n",
              " 3003132: '3376-1',\n",
              " 3002371: '4935-3',\n",
              " 3002440: '14062-4',\n",
              " 3002549: '11878-6',\n",
              " 3002568: '2269-9',\n",
              " 3002140: '4478-4',\n",
              " 3002209: '11884-4',\n",
              " 3002246: '15915-2',\n",
              " 3002253: '5813-1',\n",
              " 3002303: '6914-6',\n",
              " 3002400: '2498-4',\n",
              " 3002447: '22446-9',\n",
              " 3003101: '25303-9',\n",
              " 3003172: '32364-2',\n",
              " 3003236: '31106-8',\n",
              " 3003245: '4477-6',\n",
              " 3003455: '31406-2',\n",
              " 3003467: '26476-2',\n",
              " 3003555: '5734-9',\n",
              " 3003573: '948-0',\n",
              " 3003621: '24039-0',\n",
              " 3002661: '18996-9',\n",
              " 3002731: '6920-3',\n",
              " 3002762: '2961-1',\n",
              " 3003471: '6913-8',\n",
              " 3003564: '16257-8',\n",
              " 3003715: '7792-5',\n",
              " 3002730: '3755-6',\n",
              " 3003867: '14211-7',\n",
              " 3003898: '10585-8',\n",
              " 3003977: '14315-6',\n",
              " 3003993: '1434-0',\n",
              " 3003994: '32546-4',\n",
              " 3004501: '2345-7',\n",
              " 3004944: '3714-3',\n",
              " 3003424: '11281-3',\n",
              " 3004339: '4476-8',\n",
              " 3004820: '2240-0',\n",
              " 3004955: '20660-7',\n",
              " 3004977: '6193-7',\n",
              " 3005005: '30193-7',\n",
              " 3005058: '19270-8',\n",
              " 3005081: '4625-0',\n",
              " 3005114: '15684-4',\n",
              " 3005167: '3831-5',\n",
              " 3003516: '6083-0',\n",
              " 3003541: '12611-0',\n",
              " 3003545: '25610-7',\n",
              " 3003603: '2196-4',\n",
              " 3003645: '5060-9',\n",
              " 3003765: '6161-4',\n",
              " 3004217: '27387-0',\n",
              " 3004257: '15802-2',\n",
              " 3004269: '25682-6',\n",
              " 3004295: '6299-2',\n",
              " 3004327: '731-0',\n",
              " 3004395: '8670-2',\n",
              " 3004409: '3218-5',\n",
              " 3004410: '4548-4',\n",
              " 3004578: '26848-2',\n",
              " 3004665: '32552-2',\n",
              " 3004330: '3983-4',\n",
              " 3004404: '17815-2',\n",
              " 3004706: '20654-0',\n",
              " 3004722: '2842-3',\n",
              " 3014955: '1764-0',\n",
              " 3015737: '29142-7',\n",
              " 3015746: '31208-2',\n",
              " 3006047: '26601-5',\n",
              " 3006133: '11093-2',\n",
              " 3016431: '19072-8',\n",
              " 3016447: '2905-8',\n",
              " 3016641: '29522-0',\n",
              " 3016699: '1504-0',\n",
              " 3016716: '9645-3',\n",
              " 3016771: '1798-8',\n",
              " 3005131: '27353-2',\n",
              " 3005279: '3897-6',\n",
              " 3005323: '5726-5',\n",
              " 3005375: '7248-8',\n",
              " 3015173: '460-6',\n",
              " 3015239: '12556-7',\n",
              " 3015397: '13733-1',\n",
              " 3015411: '15650-5',\n",
              " 3015322: '2865-4',\n",
              " 3015375: '3330-8',\n",
              " 3015418: '21001-3',\n",
              " 3015497: '6112-7',\n",
              " 3015597: '2492-7',\n",
              " 3014997: '5723-2',\n",
              " 3015057: '22466-7',\n",
              " 3015818: '5077-3',\n",
              " 3015883: '12293-7',\n",
              " 3015923: '17726-1',\n",
              " 3015944: '22211-7',\n",
              " 3015982: '6182-0',\n",
              " 3016121: '13591-3',\n",
              " 3016133: '20573-2',\n",
              " 3014727: '21365-2',\n",
              " 3014814: '3779-6',\n",
              " 3015077: '9413-6',\n",
              " 3005738: '27389-6',\n",
              " 3005755: '1743-4',\n",
              " 3005791: '25638-8',\n",
              " 3005826: '2567-6',\n",
              " 3006015: '22708-2',\n",
              " 3006029: '29519-6',\n",
              " 3015924: '4459-4',\n",
              " 3006451: '6273-7',\n",
              " 3006467: '30195-2',\n",
              " 3006576: '1959-6',\n",
              " 3006594: '15647-1',\n",
              " 3006806: '30201-8',\n",
              " 3006840: '6122-6',\n",
              " 3017177: '5232-4',\n",
              " 3017205: '27085-0',\n",
              " 3017215: '19074-4',\n",
              " 3017259: '13966-7',\n",
              " 3017278: '19145-2',\n",
              " 3017295: '21374-4',\n",
              " 3032828: '41618-0',\n",
              " 3033396: '42804-5',\n",
              " 3017675: '29893-5',\n",
              " 3017766: '4498-2',\n",
              " 3017792: '15641-4',\n",
              " 3008228: '6115-0',\n",
              " 3008269: '5622-6',\n",
              " 3008358: '27834-1',\n",
              " 3034452: '3393-6',\n",
              " 3006977: '12945-2',\n",
              " 3006997: '2988-4',\n",
              " 3007034: '21305-8',\n",
              " 3007107: '22707-4',\n",
              " 3007128: '29518-8',\n",
              " 3007171: '5623-4',\n",
              " 3007173: '18878-9',\n",
              " 3007326: '18047-1',\n",
              " 3007350: '8246-1',\n",
              " 3016835: '15936-8',\n",
              " 3016914: '606-4',\n",
              " 3016941: '9339-3',\n",
              " 3017043: '2605-4',\n",
              " 3017044: '5385-0',\n",
              " 3017165: '6261-2',\n",
              " 3017935: '21217-5',\n",
              " 3007487: '3950-3',\n",
              " 3007518: '6266-1',\n",
              " 3007757: '6109-3',\n",
              " 3033661: '29500-6',\n",
              " 3033682: '5358-7',\n",
              " 3007843: '25107-4',\n",
              " 3007887: '6200-0',\n",
              " 3007913: '19991-9',\n",
              " 3008090: '2218-6',\n",
              " 3008122: '11526-1',\n",
              " 3008140: '6412-1',\n",
              " 3008142: '18337-6',\n",
              " 3006407: '9811-1',\n",
              " 3006570: '27852-3',\n",
              " 3006572: '30038-4',\n",
              " 3006581: '23658-8',\n",
              " 3006678: '1815-0',\n",
              " 3006790: '5696-0',\n",
              " 3007027: '14251-3',\n",
              " 3007035: '16074-7',\n",
              " 3016723: '2160-0',\n",
              " 3016942: '24462-4',\n",
              " 3033654: '21347-0',\n",
              " 3007854: '20655-7',\n",
              " 3007856: '23820-4',\n",
              " 3007857: '3872-9',\n",
              " 3007858: '2542-9',\n",
              " 3007876: '5767-9',\n",
              " 3007953: '21383-5',\n",
              " 3007956: '9425-0',\n",
              " 3035124: '13945-1',\n",
              " 3008625: '2230-1',\n",
              " 3017872: '21074-0',\n",
              " 3018001: '6190-3',\n",
              " 3018039: '30190-3',\n",
              " 3018056: '4269-7',\n",
              " 3018081: '1953-9',\n",
              " 3034533: '11141-9',\n",
              " 3034548: '35741-8',\n",
              " 3034577: '38162-4',\n",
              " 3034600: '36897-7',\n",
              " 3034660: '41994-5',\n",
              " 3034782: '21348-8',\n",
              " 3034812: '9625-5',\n",
              " 3034913: '35318-5',\n",
              " 3009628: '17351-8',\n",
              " 3009682: '2143-6',\n",
              " 3009745: '15975-6',\n",
              " 3035043: '3537-8',\n",
              " 3035069: '16130-7',\n",
              " 3019419: '12241-6',\n",
              " 3050930: '53842-1',\n",
              " 3050637: '53774-6',\n",
              " 3035820: '35534-7',\n",
              " 3018174: '3033-8',\n",
              " 3018199: '26507-4',\n",
              " 3016397: '22698-5',\n",
              " 3016545: '25135-5',\n",
              " 3016555: '2407-5',\n",
              " 3016648: '18719-5',\n",
              " 3007124: '17849-1',\n",
              " 3007368: '2464-6',\n",
              " 3033347: '26028-1',\n",
              " 3017509: '3936-2',\n",
              " 3017721: '30328-9',\n",
              " 3017848: '29901-6',\n",
              " 3017962: '21753-9',\n",
              " 3007534: '32101-8',\n",
              " 3007591: '764-1',\n",
              " 3007682: '14316-4',\n",
              " 3007687: '6874-2',\n",
              " 3007714: '14924-5',\n",
              " 3033543: '2965-2',\n",
              " 3033564: '16960-7',\n",
              " 3033604: '24435-0',\n",
              " 3033621: '15388-2',\n",
              " 3033641: '9317-9',\n",
              " 3033810: '25086-0',\n",
              " 3033840: '972-0',\n",
              " 3033851: '1942-2',\n",
              " 3007844: '5159-9',\n",
              " 3008009: '3175-7',\n",
              " 3008091: '3548-5',\n",
              " 3008211: '22706-6',\n",
              " 3022341: '9361-7',\n",
              " 3022374: '30196-0',\n",
              " 3022460: '15632-3',\n",
              " 3012028: '27270-8',\n",
              " 3012056: '14945-0',\n",
              " 3012180: '7091-2',\n",
              " 3012336: '4542-7',\n",
              " 3009088: '17850-9',\n",
              " 3009220: '13654-9',\n",
              " 3009249: '10907-4',\n",
              " 3009261: '25428-4',\n",
              " 3009401: '20629-2',\n",
              " 3009417: '2112-1',\n",
              " 3050252: '38404-0',\n",
              " 3008217: '5923-8',\n",
              " 3008325: '12258-0',\n",
              " 3008342: '770-8',\n",
              " 3008440: '21215-9',\n",
              " 3008477: '29356-3',\n",
              " 3008576: '14083-0',\n",
              " 3018478: '5765-3',\n",
              " 3009563: '19370-6',\n",
              " 3009687: '26574-4',\n",
              " 3009754: '6065-7',\n",
              " 3009883: '6220-8',\n",
              " 3018572: '2069-3',\n",
              " 3018595: '23883-2',\n",
              " 3036164: '29503-0',\n",
              " 3036339: '428-3',\n",
              " 3036517: '11221-9',\n",
              " 3018994: '14803-1',\n",
              " 3019041: '13997-2',\n",
              " 3010623: '1987-7',\n",
              " 3010625: '14044-2',\n",
              " 3010653: '6138-2',\n",
              " 3010657: '29239-1',\n",
              " 3010836: '15967-3',\n",
              " 3011001: '5650-7',\n",
              " 3011003: '11257-3',\n",
              " 3019909: '4545-0',\n",
              " 3020022: '14208-3',\n",
              " 3020080: '27882-0',\n",
              " 3020138: '14118-4',\n",
              " 3036774: '5975-8',\n",
              " 3036955: '15015-1',\n",
              " 3036987: '2284-8',\n",
              " 3037142: '29575-8',\n",
              " 3023479: '27810-1',\n",
              " 3012885: '26575-1',\n",
              " 3012920: '5643-2',\n",
              " 3012963: '14114-3',\n",
              " 3008882: '15952-5',\n",
              " 3008909: '7974-9',\n",
              " 3009003: '26583-5',\n",
              " 3018057: '5599-6',\n",
              " 3018069: '10352-3',\n",
              " 3018143: '5157-3',\n",
              " 3018171: '19080-1',\n",
              " 3018276: '10884-5',\n",
              " 3018356: '15943-4',\n",
              " 3008708: '6042-6',\n",
              " 3008804: '1496-9',\n",
              " 3008898: '4821-5',\n",
              " 3009046: '18271-7',\n",
              " 3018127: '12783-7',\n",
              " 3010414: '23992-1',\n",
              " 3010493: '3128-6',\n",
              " 3020960: '30170-5',\n",
              " 3021009: '4552-6',\n",
              " 3021044: '2500-7',\n",
              " 3037207: '9710-5',\n",
              " 3037215: '6918-7',\n",
              " 3037381: '6167-1',\n",
              " 3037436: '22692-8',\n",
              " 3037534: '2729-2',\n",
              " 3037553: '2127-9',\n",
              " 3037563: '3457-9',\n",
              " 3021097: '5116-9',\n",
              " 3021236: '5370-2',\n",
              " 3021257: '24349-3',\n",
              " 3021347: '1995-0',\n",
              " 3021483: '15554-9',\n",
              " 3021546: '6223-2',\n",
              " 3011051: '25134-8',\n",
              " 3011205: '14335-4',\n",
              " 3011306: '5722-4',\n",
              " 3038162: '9474-8',\n",
              " 3013642: '3768-9',\n",
              " 3013731: '5193-8',\n",
              " 3013735: '12515-3',\n",
              " 3013801: '13955-0',\n",
              " 3013859: '22131-7',\n",
              " 3013931: '12239-0',\n",
              " 3013983: '19586-7',\n",
              " 3040072: '40478-0',\n",
              " 3014033: '6288-5',\n",
              " 3014037: '8123-2',\n",
              " 3014053: '2341-6',\n",
              " 3014071: '5043-5',\n",
              " 3014126: '6110-1',\n",
              " 3010913: '13829-7',\n",
              " 3036814: '587-6',\n",
              " 3036947: '31037-5',\n",
              " 3036949: '7573-9',\n",
              " 3037010: '38169-9',\n",
              " 3037064: '35802-8',\n",
              " 3010454: '11034-6',\n",
              " 3010456: '3592-3',\n",
              " 3010483: '1305-2',\n",
              " 3010509: '13464-3',\n",
              " 3010702: '15680-2',\n",
              " 3050388: '46217-6',\n",
              " 3010137: '29623-6',\n",
              " 3010190: '3127-8',\n",
              " 3010228: '25133-0',\n",
              " 3010412: '15981-4',\n",
              " 3010422: '6071-5',\n",
              " 3010457: '713-8',\n",
              " 3010816: '11099-9',\n",
              " 3010912: '29620-2',\n",
              " 3010955: '30056-6',\n",
              " 3010981: '8267-7',\n",
              " 3011105: '12207-7',\n",
              " 3011172: '5335-5',\n",
              " 3037043: '38178-0',\n",
              " 3037167: '664-3',\n",
              " 3037172: '15853-5',\n",
              " 3037201: '27823-4',\n",
              " 3018646: '14312-3',\n",
              " 3018809: '25099-3',\n",
              " 3018857: '3098-1',\n",
              " 3035283: '42233-7',\n",
              " 3035362: '15697-6',\n",
              " 3035380: '6100-2',\n",
              " 3035554: '15550-7',\n",
              " 3051287: '46429-7',\n",
              " 3036324: '5070-8',\n",
              " 3036343: '30237-2',\n",
              " 3036413: '35475-3',\n",
              " 3036474: '21428-8',\n",
              " 3036524: '26880-5',\n",
              " 3036545: '6166-3',\n",
              " 3036607: '22691-0',\n",
              " 3052635: '48390-9',\n",
              " 3035967: '22685-2',\n",
              " 3036005: '11279-7',\n",
              " 3036049: '15545-7',\n",
              " 3019800: '6598-7',\n",
              " 3019809: '27039-7',\n",
              " 3019813: '18908-4',\n",
              " 3051288: '46985-8',\n",
              " 3023512: '16031-7',\n",
              " 3023520: '14196-0',\n",
              " 3023574: '9833-5',\n",
              " 3023640: '2258-2',\n",
              " 3023709: '2963-7',\n",
              " 3013184: '19659-2',\n",
              " 3036669: '27822-6',\n",
              " 3036788: '9633-9',\n",
              " 3019899: '15977-2',\n",
              " 3019902: '13317-3',\n",
              " 3019978: '16131-5',\n",
              " 3019994: '6092-1',\n",
              " 3020211: '21563-2',\n",
              " 3020229: '5000-5',\n",
              " 3010023: '21026-0',\n",
              " 3010024: '1078-5',\n",
              " 3035774: '35388-8',\n",
              " 3035801: '35384-7',\n",
              " 3035845: '2142-8',\n",
              " 3035964: '13344-7',\n",
              " 3036273: '7797-4',\n",
              " 3036506: '29497-5',\n",
              " 3009956: '19429-0',\n",
              " 3010007: '185-9',\n",
              " 3010074: '29539-4',\n",
              " 3010109: '5644-0',\n",
              " 3010189: '5787-7',\n",
              " 3010293: '10604-7',\n",
              " 3010297: '3282-1',\n",
              " 3010375: '3520-4',\n",
              " 3010391: '14115-0',\n",
              " 3020389: '10701-1',\n",
              " 3020457: '5273-8',\n",
              " 3020475: '5646-5',\n",
              " 3020527: '29776-2',\n",
              " 3020630: '2885-2',\n",
              " 3020646: '15616-6',\n",
              " 3020712: '31097-9',\n",
              " 3051256: '53596-3',\n",
              " 3051367: '48019-4',\n",
              " 3051383: '48084-8',\n",
              " 3036437: '35300-3',\n",
              " 3036571: '5372-8',\n",
              " 3036592: '7195-1',\n",
              " 3036616: '6917-9',\n",
              " 3036727: '38177-2',\n",
              " 3036753: '35789-7',\n",
              " 3019869: '9834-3',\n",
              " 3020029: '5809-9',\n",
              " 3020174: '21027-8',\n",
              " 3024303: '3126-0',\n",
              " 3052043: '48305-7',\n",
              " 3052053: '48311-5',\n",
              " 3050693: '46758-9',\n",
              " 3035706: '15539-0',\n",
              " 3035962: '31201-7',\n",
              " 3036008: '15774-3',\n",
              " 3036035: '27816-8',\n",
              " 3019323: '32576-1',\n",
              " 3019347: '6825-4',\n",
              " 3019509: '3746-5',\n",
              " 3051014: '46702-7',\n",
              " 3036067: '15846-9',\n",
              " 3036230: '9627-1',\n",
              " 3036243: '2821-7',\n",
              " 3036285: '20633-4',\n",
              " 3036361: '16038-2',\n",
              " 3019641: '30518-5',\n",
              " 3019736: '6694-4',\n",
              " 3020897: '6848-6',\n",
              " 3020990: '15014-4',\n",
              " 3010684: '30209-1',\n",
              " 3010747: '20447-9',\n",
              " 3010785: '28588-2',\n",
              " 3010833: '28593-2',\n",
              " 3010875: '14035-0',\n",
              " 3010910: '26455-6',\n",
              " 3010911: '25125-6',\n",
              " 3010645: '27416-7',\n",
              " 3010807: '3363-9',\n",
              " 3036662: '6171-3',\n",
              " 3036691: '35673-3',\n",
              " 3036809: '20634-2',\n",
              " 3036824: '21362-9',\n",
              " 3036834: '6239-8',\n",
              " 3036882: '5817-2',\n",
              " 3019961: '11071-8',\n",
              " 3036265: '4644-1',\n",
              " 3036277: '8302-2',\n",
              " 3036325: '13722-4',\n",
              " 3019406: '6158-0',\n",
              " 3019531: '5568-1',\n",
              " 3019561: '28067-7',\n",
              " 3019660: '22670-4',\n",
              " 3019742: '7187-8',\n",
              " 3019761: '7795-8',\n",
              " 3020509: '1759-0',\n",
              " 3020650: '2349-9',\n",
              " 3020695: '15749-5',\n",
              " 3020740: '8198-4',\n",
              " 3020763: '15984-8',\n",
              " 3019785: '21558-2',\n",
              " 3019812: '11050-2',\n",
              " 3019884: '2722-7',\n",
              " 3019930: '3093-2',\n",
              " 3019965: '19049-6',\n",
              " 3019974: '32081-2',\n",
              " 3020536: '287-3',\n",
              " 3020545: '3924-8',\n",
              " 3020946: '5213-4',\n",
              " 3020963: '662-7',\n",
              " 3020978: '32366-7',\n",
              " 3020994: '31108-4',\n",
              " 3021320: '6447-7',\n",
              " 3021332: '12602-9',\n",
              " 3021337: '10839-9',\n",
              " 3021438: '27854-9',\n",
              " 3052558: '46499-0',\n",
              " 3052599: '48501-1',\n",
              " 3021461: '20507-0',\n",
              " 3021650: '30519-3',\n",
              " 3021717: '3050-2',\n",
              " 3021785: '25087-8',\n",
              " 3011483: '5793-5',\n",
              " 3011519: '21017-9',\n",
              " 3011520: '24110-9',\n",
              " 3011547: '3198-9',\n",
              " 3011562: '26612-2',\n",
              " 3037503: '20398-4',\n",
              " 3037539: '17819-4',\n",
              " 3037544: '2892-8',\n",
              " 3037585: '2428-1',\n",
              " 3037692: '593-4',\n",
              " 3037791: '13801-6',\n",
              " 3037932: '42985-2',\n",
              " 3021374: '29247-4',\n",
              " 3021532: '13502-0',\n",
              " 3021534: '27831-7',\n",
              " 3052183: '47359-5',\n",
              " 3037466: '15016-9',\n",
              " 3037467: '24357-6',\n",
              " 3037475: '6244-8',\n",
              " 3037533: '6014-5',\n",
              " 3037574: '18475-4',\n",
              " 3037756: '25700-6',\n",
              " 3018978: '23920-2',\n",
              " 3019137: '18860-7',\n",
              " 3010300: '20438-8',\n",
              " 3010642: '7095-3',\n",
              " 3051714: '48065-7',\n",
              " 3051971: '47527-7',\n",
              " 3020331: '5671-3',\n",
              " 3037057: '35393-8',\n",
              " 3037060: '36922-3',\n",
              " 3037123: '3321-7',\n",
              " 3021763: '5095-5',\n",
              " 3021767: '9590-1',\n",
              " 3021769: '25416-9',\n",
              " 3021770: '26746-8',\n",
              " 3052587: '48505-2',\n",
              " 3052684: '48306-5',\n",
              " 3011298: '635-3',\n",
              " 3011335: '10535-3',\n",
              " 3011360: '16016-8',\n",
              " 3011402: '19554-5',\n",
              " 3011508: '30358-6',\n",
              " 3011523: '7458-3',\n",
              " 3011564: '5334-8',\n",
              " 3037950: '42772-4',\n",
              " 3038062: '23937-6',\n",
              " 3038117: '9776-6',\n",
              " 3038126: '30551-6',\n",
              " 3038160: '19648-5',\n",
              " 3037282: '13788-5',\n",
              " 3037329: '36923-1',\n",
              " 3052708: '48495-6',\n",
              " 3022010: '15761-0',\n",
              " ...}"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# create map to later add LOINCids to the 2020 demographics DF\n",
        "loincMap = dict(zip(list(omc['OMOPid']), list(omc['LOINCid'])))\n",
        "loincMap"
      ],
      "id": "f2a33474"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "id": "98421fd1",
        "outputId": "0646479f-c39f-4969-cefe-cbdf4d421b3b",
        "scrolled": true
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-29-4dd0c18e106e>:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  filtered['LOINCid'] = [loincMap[filtered['OMOPid'][omop]] for omop in range(len(filtered))]\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4a91d4e5-3510-4e7d-94a5-9e151c8e5f0e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3010421</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>NaN</td>\n",
              "      <td>7.35</td>\n",
              "      <td>7.45</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11558-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm Hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dL</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "      <td>2160-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "      <td>6690-2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "      <td>43743-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10E9/L</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "      <td>731-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "      <td>25132-2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14244786 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4a91d4e5-3510-4e7d-94a5-9e151c8e5f0e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4a91d4e5-3510-4e7d-94a5-9e151c8e5f0e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4a91d4e5-3510-4e7d-94a5-9e151c8e5f0e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer            unit  range_low  range_high  gender  \\\n",
              "0         3010421      LAB5808             NaN       7.35        7.45  Female   \n",
              "1         3013502      LAB5808               %      95.00       99.00  Female   \n",
              "2         3013502      LAB5809               %      95.00       99.00  Female   \n",
              "3         3027315      LAB5808           mm Hg      80.00      100.00  Female   \n",
              "4         3027315      LAB5809           mm Hg      80.00      100.00  Female   \n",
              "...           ...          ...             ...        ...         ...     ...   \n",
              "14244781  3016723      LAB3001           mg/dL       0.61        1.24    Male   \n",
              "14244782  3000905       LAB294         x10E9/L       3.40       10.00  Female   \n",
              "14244783  3046553       LAB293         x10E9/L       0.00        0.20  Female   \n",
              "14244784  3004327       LAB293         x10E9/L       0.90        2.90    Male   \n",
              "14244785  3023996       LC3743  mmol/mol creat       1.00       14.00    Male   \n",
              "\n",
              "                       ethnicity                race  age  LOINCid  \n",
              "0         Not Hispanic or Latino               Asian    0  11558-4  \n",
              "1         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "2         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "3         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "4         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "...                          ...                 ...  ...      ...  \n",
              "14244781      Hispanic or Latino  White or Caucasian   53   2160-0  \n",
              "14244782                 Unknown             Unknown   71   6690-2  \n",
              "14244783      Hispanic or Latino  White or Caucasian    2  43743-4  \n",
              "14244784  Not Hispanic or Latino  White or Caucasian   36    731-0  \n",
              "14244785      Hispanic or Latino             Unknown   13  25132-2  \n",
              "\n",
              "[14244786 rows x 10 columns]"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# add Loinc IDs to demographic DF\n",
        "filtered['LOINCid'] = [loincMap[filtered['OMOPid'][omop]] for omop in range(len(filtered))]\n",
        "filtered"
      ],
      "id": "98421fd1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b3a5cd09",
        "outputId": "ca90a44b-ce27-45b2-8386-e0b33b424ebc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "range is 0-0 for the following units:\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'%', 'PERCENT', 'mmol/mol creat'}"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"range is 0-0 for the following units:\")\n",
        "set(filtered[(filtered['range_low']== 0) & (filtered['range_high']== 0)]['unit'])"
      ],
      "id": "b3a5cd09"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "id": "2394439d",
        "outputId": "1cc0d33d-37ac-4db4-bfd8-b8688cab347e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-31-787c85b7c6d8>:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  filtered['unit'] = [str(unit).lower().strip() for unit in filtered['unit']]\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-696e96d9-8c70-46ad-8dfc-25b7cb356f0d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3010421</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>nan</td>\n",
              "      <td>7.35</td>\n",
              "      <td>7.45</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11558-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "      <td>2160-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "      <td>6690-2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "      <td>43743-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "      <td>731-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "      <td>25132-2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14244786 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-696e96d9-8c70-46ad-8dfc-25b7cb356f0d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-696e96d9-8c70-46ad-8dfc-25b7cb356f0d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-696e96d9-8c70-46ad-8dfc-25b7cb356f0d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer            unit  range_low  range_high  gender  \\\n",
              "0         3010421      LAB5808             nan       7.35        7.45  Female   \n",
              "1         3013502      LAB5808               %      95.00       99.00  Female   \n",
              "2         3013502      LAB5809               %      95.00       99.00  Female   \n",
              "3         3027315      LAB5808           mm hg      80.00      100.00  Female   \n",
              "4         3027315      LAB5809           mm hg      80.00      100.00  Female   \n",
              "...           ...          ...             ...        ...         ...     ...   \n",
              "14244781  3016723      LAB3001           mg/dl       0.61        1.24    Male   \n",
              "14244782  3000905       LAB294         x10e9/l       3.40       10.00  Female   \n",
              "14244783  3046553       LAB293         x10e9/l       0.00        0.20  Female   \n",
              "14244784  3004327       LAB293         x10e9/l       0.90        2.90    Male   \n",
              "14244785  3023996       LC3743  mmol/mol creat       1.00       14.00    Male   \n",
              "\n",
              "                       ethnicity                race  age  LOINCid  \n",
              "0         Not Hispanic or Latino               Asian    0  11558-4  \n",
              "1         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "2         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "3         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "4         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "...                          ...                 ...  ...      ...  \n",
              "14244781      Hispanic or Latino  White or Caucasian   53   2160-0  \n",
              "14244782                 Unknown             Unknown   71   6690-2  \n",
              "14244783      Hispanic or Latino  White or Caucasian    2  43743-4  \n",
              "14244784  Not Hispanic or Latino  White or Caucasian   36    731-0  \n",
              "14244785      Hispanic or Latino             Unknown   13  25132-2  \n",
              "\n",
              "[14244786 rows x 10 columns]"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# convert units to lowercase\n",
        "filtered['unit'] = [str(unit).lower().strip() for unit in filtered['unit']]\n",
        "filtered"
      ],
      "id": "2394439d"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9204e17c"
      },
      "source": [
        "# Situations in which labs tests don't have clear ranges\n",
        "1. range is not a number\n",
        "\n",
        "    - DONE Result: All are floats.\n",
        "\n",
        "\n",
        "2. range does not have reasonable low or high value\n",
        "\n",
        "    2a) for [low, high], low > high\n",
        "    \n",
        "        - DONE Result: 121 rows (err_range)\n",
        "        \n",
        "    2b) low < 0 or high < 0\n",
        "    \n",
        "        - DONE Result: check passed (0 match this) (neg_low_high)\n",
        "    \n",
        "3. range is different for same manufacturer, omopid, and demographic\n",
        "\n",
        "        - DONE Progress: There exist ranges that are different despite being from same manufacturer, omopid, and demographic.\n",
        "            - Charlotte's suggestion: Could add year it was taken in case the range was updated later on. However, might be a better option to just add a confidence interval.\n",
        "            - my approach: Apply confidence interval. If difference is beyond the CI, add both to a separate unclear ranges table (\"conflicting_ranges_df\", so its easier to find what about it makes it unclear.\n",
        "    ***Progress:***\n",
        "\n",
        "                - implemented function to convert to \"any\" if meets condition (condition being that all values for this demographic are present and the range is the same)\n",
        "                    - need to check with Charlotte if this is the way she intends this to be implemented (DONE)\n",
        "  \n",
        "                   \n",
        "        \n",
        "4. unit is unclear (nan,  'not reported', 'index', 'units', 'undex', 'index value', '(calc)')\n",
        "    - DONE Result: 372,650 rows\n",
        "    \n",
        "5. age is negative\n",
        "    - DONE Result: 90 rows"
      ],
      "id": "9204e17c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6f494878",
        "outputId": "a9895fee-c3c9-4b8d-ba89-caea510b3588"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sum(list(np.isnan(filtered['range_high'].values)))"
      ],
      "id": "6f494878"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LZBVJ4CtIOXn"
      },
      "source": [
        "## Situations in which lab tests don't have clear ranges (CODE)"
      ],
      "id": "LZBVJ4CtIOXn"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dbfaf93a"
      },
      "source": [
        "1. range is not a number\n",
        "\n",
        "    - DONE Result: All are floats."
      ],
      "id": "dbfaf93a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fdadbdf6",
        "outputId": "6badea58-d83e-4ad2-881c-f14d0310d147"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of rows where range_low is not a float or NaN:\n",
            " 0\n",
            "Number of rows where range_high is not a float or NaN:\n",
            " 0\n"
          ]
        }
      ],
      "source": [
        "# 1. range is not a float & not NaN\n",
        "typelow = sum(list(filtered['range_low'].map(type) != float) or list(np.isnan(filtered['range_low'].values)))\n",
        "typehigh = sum(list(filtered['range_high'].map(type) != float) or list(np.isnan(filtered['range_high'].values)))\n",
        "print('Number of rows where range_low is not a float or NaN:\\n', typelow)\n",
        "print('Number of rows where range_high is not a float or NaN:\\n', typehigh)\n",
        "# all are floats."
      ],
      "id": "fdadbdf6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cb5c56c5"
      },
      "source": [
        "2. range does not have reasonable low or high value\n",
        "\n",
        "    2a) for [low, high], low > high\n",
        "    \n",
        "        - DONE Result: 121 rows (errRange)\n",
        "        \n",
        "    2b) low < 0 or high < 0\n",
        "    \n",
        "        - DONE Result: check passed (0 match this) (neg_low_high)"
      ],
      "id": "cb5c56c5"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "29a5f3bc",
        "outputId": "c7cdf770-b6e9-4476-96c6-c0db8a0ba0b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of rows where range_low > range_high:\n",
            " 121\n",
            "Number of rows where range_low < 0 or range_high < 0:\n",
            " 0\n"
          ]
        }
      ],
      "source": [
        "# 2a. for [low, high], low >= high\n",
        "err_range = filtered[filtered['range_low'] > filtered['range_high']]\n",
        "# append unclear indeces to to_drop\n",
        "to_drop = list(err_range.index)\n",
        "\n",
        "print(\"Number of rows where range_low > range_high:\\n\", str(len(err_range)))\n",
        "\n",
        "\n",
        "# 2b. low or high < 0\n",
        "neg_low_high = filtered[(filtered['range_low'] < 0) | (filtered['range_high'] < 0)]\n",
        "\n",
        "print(\"Number of rows where range_low < 0 or range_high < 0:\\n\", str(len(neg_low_high)))\n",
        "\n",
        "\n",
        "# Dont need to check if range_high = 0, since a low, high range = 0,0 is valid in cases where patients should have 0 concentration of something.\n",
        "# invalid_high = filtered[(filtered['range_high'] <= 0)]\n",
        "# print(\"Number of rows where range_high <= 0:\\n\", str(len(invalid_high)))"
      ],
      "id": "29a5f3bc"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2bac0d3b"
      },
      "source": [
        "3. range is different for same manufacturer, omopid, and demographic\n",
        "\n",
        "See below"
      ],
      "id": "2bac0d3b"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25c4f6a1"
      },
      "source": [
        "4. unit is unclear ('not reported', 'index', 'units', 'index value', '(calc)', 'nan', None)\n",
        "    - DONE Result: 372,650 rows"
      ],
      "id": "25c4f6a1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f12c841a",
        "outputId": "cd3507b8-07f6-4eb6-c629-c2bcd7bea67e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of rows with unclear units (''not reported', 'index', 'units', 'index value', '(calc)', 'nan', None:\n",
            "372650\n"
          ]
        }
      ],
      "source": [
        "# 4. unit is unclear\n",
        "unclear = ['not reported', 'index', 'units', 'index value', '(calc)', 'nan', None]\n",
        "\n",
        "uncleardf = filtered[filtered['unit'].isin(unclear)]\n",
        "# append unclear indeces to to_drop\n",
        "to_drop += list(uncleardf.index)\n",
        "\n",
        "nanDF = filtered[filtered['unit'].isnull()]\n",
        "# append unclear indeces to to_drop\n",
        "to_drop += list(nanDF.index)\n",
        "\n",
        "print(\"Number of rows with unclear units (''not reported', 'index', 'units', 'index value', '(calc)', 'nan', None):\")\n",
        "print(len(uncleardf) + len(nanDF))"
      ],
      "id": "f12c841a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5f595bf1"
      },
      "source": [
        "5. age is negative\n",
        "    - DONE Result: 90 rows"
      ],
      "id": "5f595bf1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        },
        "id": "598114b4",
        "outputId": "53a92020-77c6-4578-9d3c-816fab7a377f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of rows with age value < 0:\n",
            "90\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-0875cef3-fdf9-4fe0-b31c-cfe87891aabf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1295363</th>\n",
              "      <td>3018572</td>\n",
              "      <td>LAB3003</td>\n",
              "      <td>mmol/l</td>\n",
              "      <td>98.0</td>\n",
              "      <td>108.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>-99</td>\n",
              "      <td>2069-3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1295365</th>\n",
              "      <td>3013290</td>\n",
              "      <td>LAB3003</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>35.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>-99</td>\n",
              "      <td>11557-6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1295396</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB3003</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.0</td>\n",
              "      <td>100.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>-99</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2269900</th>\n",
              "      <td>3033575</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.1</td>\n",
              "      <td>0.6</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>-5</td>\n",
              "      <td>742-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2269901</th>\n",
              "      <td>3000905</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>3.4</td>\n",
              "      <td>10.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>-5</td>\n",
              "      <td>6690-2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14170110</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB3239</td>\n",
              "      <td>%</td>\n",
              "      <td>95.0</td>\n",
              "      <td>99.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>-1</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14170111</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB3239</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.0</td>\n",
              "      <td>100.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>-1</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14170112</th>\n",
              "      <td>3013290</td>\n",
              "      <td>LAB3239</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>27.0</td>\n",
              "      <td>41.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>-1</td>\n",
              "      <td>11557-6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14170113</th>\n",
              "      <td>3024882</td>\n",
              "      <td>LAB3239</td>\n",
              "      <td>%</td>\n",
              "      <td>20.0</td>\n",
              "      <td>100.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>-1</td>\n",
              "      <td>19994-3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14170114</th>\n",
              "      <td>3013290</td>\n",
              "      <td>LAB3239</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>27.0</td>\n",
              "      <td>41.0</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>-1</td>\n",
              "      <td>11557-6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>90 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0875cef3-fdf9-4fe0-b31c-cfe87891aabf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0875cef3-fdf9-4fe0-b31c-cfe87891aabf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0875cef3-fdf9-4fe0-b31c-cfe87891aabf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer     unit  range_low  range_high gender  \\\n",
              "1295363   3018572      LAB3003   mmol/l       98.0       108.0   Male   \n",
              "1295365   3013290      LAB3003    mm hg       35.0        45.0   Male   \n",
              "1295396   3027315      LAB3003    mm hg       80.0       100.0   Male   \n",
              "2269900   3033575       LAB293  x10e9/l        0.1         0.6   Male   \n",
              "2269901   3000905       LAB293  x10e9/l        3.4        10.0   Male   \n",
              "...           ...          ...      ...        ...         ...    ...   \n",
              "14170110  3013502      LAB3239        %       95.0        99.0   Male   \n",
              "14170111  3027315      LAB3239    mm hg       80.0       100.0   Male   \n",
              "14170112  3013290      LAB3239    mm hg       27.0        41.0   Male   \n",
              "14170113  3024882      LAB3239        %       20.0       100.0   Male   \n",
              "14170114  3013290      LAB3239    mm hg       27.0        41.0   Male   \n",
              "\n",
              "                       ethnicity                race  age  LOINCid  \n",
              "1295363       Hispanic or Latino             Unknown  -99   2069-3  \n",
              "1295365       Hispanic or Latino             Unknown  -99  11557-6  \n",
              "1295396       Hispanic or Latino             Unknown  -99  11556-8  \n",
              "2269900   Not Hispanic or Latino  White or Caucasian   -5    742-7  \n",
              "2269901   Not Hispanic or Latino  White or Caucasian   -5   6690-2  \n",
              "...                          ...                 ...  ...      ...  \n",
              "14170110  Not Hispanic or Latino               Asian   -1  20564-1  \n",
              "14170111  Not Hispanic or Latino               Asian   -1  11556-8  \n",
              "14170112  Not Hispanic or Latino               Asian   -1  11557-6  \n",
              "14170113  Not Hispanic or Latino               Asian   -1  19994-3  \n",
              "14170114  Not Hispanic or Latino               Asian   -1  11557-6  \n",
              "\n",
              "[90 rows x 10 columns]"
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# 5. age is negative\n",
        "negAge = filtered[filtered['age'] < 0]\n",
        "# Create list of indeces to drop from filtered data frame. Will append as we check cases\n",
        "to_drop += list(negAge.index)\n",
        "\n",
        "print(\"Number of rows with age value < 0:\")\n",
        "print(len(negAge))\n",
        "negAge"
      ],
      "id": "598114b4"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4b2b9be4",
        "outputId": "b9709d86-ec16-4012-e122-b636d36d7f6a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dropping 372730 rows from df\n"
          ]
        }
      ],
      "source": [
        "\n",
        "to_drop = list(set(to_drop))\n",
        "print(\"Dropping \" + str(len(to_drop)) + \" rows from df\")"
      ],
      "id": "4b2b9be4"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b89c294b"
      },
      "outputs": [],
      "source": [
        "exp = len(filtered) - len(to_drop)"
      ],
      "id": "b89c294b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f61826ca"
      },
      "outputs": [],
      "source": [
        "# drop unclear rows from DF\n",
        "# toDrop = list(set(errRange.index.append(negAge.index).append(uncleardf.index).append(nanDF.index)))\n",
        "unclear_range = filtered.loc[to_drop]\n",
        "\n",
        "# save unclear ranges df to csv:\n",
        "# unclear_range.to_csv('unclearOMOPRanges.csv')\n",
        "\n",
        "# filter out all these unclear dfs from main df.\n",
        "clear_ranges = filtered.drop(to_drop, axis=0,inplace=False)"
      ],
      "id": "f61826ca"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "25831e06",
        "outputId": "8f50e5be-bb0d-4cfe-8c6e-462893b89848"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dropping unclear values...\n",
            "Success:  True\n"
          ]
        }
      ],
      "source": [
        "post_drop_length = len(clear_ranges)\n",
        "print(\"Dropping unclear values...\\nSuccess: \", post_drop_length == exp)"
      ],
      "id": "25831e06"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cd7e823a"
      },
      "outputs": [],
      "source": [
        "subsample = clear_ranges.head(1000)"
      ],
      "id": "cd7e823a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c15be49a"
      },
      "source": [
        "# Converting Units\n",
        "## Steps:\n",
        "### 1. Implement functions to get conversion map (unit_map) for each lab\n",
        "      a) getMaxUnit(df, c)\n",
        "      b) getUnits(df)\n",
        "### 2. Create dict with conversion multipliers for each conversion necessary\n",
        "### 3. Implement convertUnits(df)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "Want to see if units within a same lab are in a different scale.\n",
        "Brute force approach would be to make constant conversions across the board.\n",
        "\n",
        "However, we shouldn't apply conversions to all the data at once because then we might end up with values such as 95,000,000 or 0.000000095 (i.e. cant generalize which unit to convert to across labs, since for one lab it might make more sense to have the ranges in micrograms, while in another lab it might make more sense to have the unit be in milligrams).\n",
        "\n",
        "\n",
        "Need to check within each lab, which values to convert. This way we don't end up with very large or very small values, which would make us have to do another round of conversion when comparing individual results to these ranges.\n",
        "\n",
        "The functions I implemented to accomplish this task of converting units are:\n",
        "\n",
        "- getMaxUnit(df, c)\n",
        "  - inputs a df and a cutoff value.\n",
        "  - returns whether the max unit is frequent enough, the list of units present, and the max unit\n",
        "- getUnits(df)\n",
        "  - inputs a df\n",
        "  - calls getMaxUnit(df, c)\n",
        "  - returns a dictionary that maps each OMOPid to its max unit and the list of units present if a max unit is present that meats the cutoff value.\n",
        "- convertUnits(df)\n",
        "  - inputs a df\n",
        "  - returns a df with the conversions applied\n"
      ],
      "id": "c15be49a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8f0f7a61"
      },
      "source": [
        "# Examples of randomness in units used for the same lab\n",
        "Format is as follows:\n",
        "\n",
        "OMOPid: [units used for ranges in this lab]\n",
        "\n",
        "\n",
        "\n",
        "***Units that clearly don't match, unclear how to convert.***\n",
        "- Approach: Look at how dominant each unit is, set threshold to delete certain labs that don't meet it.\n",
        "\n",
        "3009744: ['g/dl', '%']\n",
        "\n",
        "3008625: ['pg/ml', 'mcg/g creat']\n",
        "\n",
        "3011337: ['ng/dl', 'ug/24 hr']\n",
        "\n",
        "3010493: ['relative to h2o', 'rel.saline']\n",
        "\n",
        "3012870: ['mg/24 hr', 'ratio']\n",
        "\n",
        "3014521: ['mcg/g creat', 'ug/d']\n",
        "\n",
        "3017575: ['u/ml', 'mg/dl', 'ng/dl', '%', 'u/l', 'mmol/l']\n",
        "\n",
        "3018478: ['mcg/24 h', 'ug/l']\n",
        "\n",
        "3020027: ['mg/24 h', 'mg in tv', 'mg/d']\n",
        "\n",
        "3020416: ['x10e6/ul',\n",
        "  '10*6/ul',\n",
        "  'm/ul',\n",
        "  'x10e6',\n",
        "  'x10e6 cells/ul',\n",
        "  'm/mm3',\n",
        "  'm/cmm',\n",
        "  'x10e12/l',\n",
        "  'million/ul',\n",
        "  '10e6/mm3',\n",
        "  '10^6/\\\\xb5l']\n",
        "  \n",
        "3024929: ['thousand/ul',\n",
        "  '10e3/mm3',\n",
        "  'x10e3',\n",
        "  'k/mm3',\n",
        "  'x10e3 cells/ul',\n",
        "  'x10e3/ul',\n",
        "  'k/\\\\xb5l',\n",
        "  't/mm3',\n",
        "  'k/ul',\n",
        "  '10*3/ul',\n",
        "  '100k',\n",
        "  'x10e9/l']\n",
        "  \n",
        "3031400: ['ug/l', 'mcg/spec']\n",
        "\n",
        "3033575: ['10e3/mm3',\n",
        "  'x10e3',\n",
        "  'x10e3 cells/ul',\n",
        "  'x10e3/ul',\n",
        "  'k/ul',\n",
        "  'cells/ul',\n",
        "  '10*3/ul',\n",
        "  'x10e9/l']\n",
        "  \n",
        "  \n",
        "3037121: ['mg/24 hr', 'mg/dl']\n",
        "  \n",
        "  \n",
        "3038191: ['umol/d', 'nm/mg cr']\n",
        "\n",
        "\n",
        "3041450: ['nmol/min/ml', 'ng/ml']\n",
        "  \n",
        "\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "\n",
        "***Examples of abnormal unit groups, diffcult to use regex on since differences are so unique:***\n",
        "\n",
        "3000067: ['ng/l', 'pg/ml']\n",
        "\n",
        "3000120: ['mg/24 h', 'mg in tv']\n",
        "\n",
        "3000620: ['mg/dl adult', 'mg/dl']\n",
        "\n",
        "3000905: ['thousand/ul',\n",
        "  'x10e3/ul',\n",
        "  'x10e3',\n",
        "  '10e3/mm3',\n",
        "  'x10e3 cells/ul',\n",
        "  'x10e9/l',\n",
        "  'k/mm3',\n",
        "  'k/ul',\n",
        "  '10*3/ul',\n",
        "  'th/mm3']\n",
        "  \n",
        "3001218: ['mg', 'mg/d', 'mg/24 h', 'mg/24 hr']\n",
        "\n",
        "3001405: ['/ul', 'cells/ul', 'x10e6/l']\n",
        "\n",
        "3001754: ['ug/dl rbcs', 'ug/dl rbc']\n",
        "\n",
        "3004239: ['g in tv', 'g/d', 'mg/d', 'mg/24 hr', 'mg/kg/d', 'g/24 h']\n",
        "\n",
        "3004327: ['x10e3',\n",
        "  'th/mm3',\n",
        "  'k/mm3',\n",
        "  'x10e3 cells/ul',\n",
        "  'x10e3/ul',\n",
        "  'k/ul',\n",
        "  'cells/ul',\n",
        "  '10*3/ul',\n",
        "  'x10e9/l']\n",
        "  \n",
        "3008295: ['mosmol/kg', 'mmol/kg', 'mosm/kg']\n",
        "\n",
        "3010493: ['relative to h2o', 'rel.saline']\n",
        "\n",
        "3023851: ['mcg/24 h', 'mcg/24h', 'ug/24 hr']\n",
        "\n",
        "3006923: ['u/l', 'iu/l']\n",
        "\n",
        "3007220: ['u/l', 'iu/l'],\n",
        "\n",
        "3009201: ['miu/l', 'uiu/ml']\n",
        "\n",
        "3017526: ['ug/l', 'mcg/g creat']\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "***Examples of obvious changes***\n",
        "\n",
        " 3002022: ['percent', '%']\n",
        "\n",
        " 3002109: ['ratio units', 'ratio']\n",
        "\n",
        " 3003308: ['% activity', '%']\n",
        "\n",
        " 3008295: ['mosmol/kg', 'mmol/kg', 'mosm/kg']\n",
        "\n",
        " 3008377: ['nmol/ml', 'umol/l']\n",
        "\n",
        " 3016948: ['sec', 's']\n",
        "\n",
        ""
      ],
      "id": "8f0f7a61"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kuDcfv6IacYb"
      },
      "source": [],
      "id": "kuDcfv6IacYb"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uyoPSqbyad49"
      },
      "source": [],
      "id": "uyoPSqbyad49"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "soarYMl8ahY3"
      },
      "source": [
        "## 1. Implement functions to get conversion map (unit_map) for each lab\n",
        "    a) getMaxUnit(df, c)\n",
        "    b) getUnits(df)"
      ],
      "id": "soarYMl8ahY3"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f9c6590a"
      },
      "outputs": [],
      "source": [
        "\n",
        "# a) getMaxUnit(df, c)\n",
        "# c is the percentage cutoff we want to use to determine whether the most frequent unit is really a significant majority of the data.\n",
        "# for example, at the last if statement in this code, it checks whether the amount of times the most frequent unit appeared is greater than the total number of observations * c.\n",
        "# So if c = 0.7, it would only count this as the most frequent unit if it has a frequency greater or equal to 70%.\n",
        "# We later experiment with different values of c to see how many clear lab ranges we can determine.\n",
        "def getMaxUnit(df, c):\n",
        "    # get unit counts\n",
        "    counts_obj = df['unit'].value_counts()\n",
        "    counts = list(counts_obj)\n",
        "    maxv = 0\n",
        "    tot = 0\n",
        "    index = 0\n",
        "    # loop through counts and determine the frequency of the most frequent unit\n",
        "    for i in range(len(counts)):\n",
        "        count = counts[i]\n",
        "        if count > maxv:\n",
        "            maxv = count\n",
        "            index = i\n",
        "        tot += count\n",
        "    # if it is frequent enough, return True, the list of existing units in this df, and the max unit\n",
        "    if maxv >= (tot*c):\n",
        "        return True, list(counts_obj.index), list(counts_obj.index)[index]\n",
        "    else:\n",
        "        return False, None, None"
      ],
      "id": "f9c6590a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4b2942da",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "from itertools import compress\n",
        "\n",
        "# b) getUnits(df)\n",
        "# Remember, entries for the same lab tend to have different units since they come from different manufacturers, and medical professionals are also just not consistent with it within the same manufacturer.\n",
        "# the purpose of this function is to take a df, and for each lab (OMOPid) find the max unit and the list of observed units used in this lab. With this, we will be able to make the conversions to the max Unit for each lab,\n",
        "# depending on the max unit for this lab, solving the issue that would arise (as described earlier) if we took a brute force approach to converting units (making the same conversion regardless of the most frequent unit).\n",
        "def getUnits(df):\n",
        "    # get unique units for each lab\n",
        "    units_obj = df.groupby('OMOPid').apply(lambda x: list(set(x['unit'])))\n",
        "    units = list(units_obj)\n",
        "\n",
        "    # get list of labs\n",
        "    omops = list(units_obj.index)\n",
        "\n",
        "    # intialize conversion dict\n",
        "    omopToUnit = {}\n",
        "    # get max unit info\n",
        "    for i in range(len(omops)):\n",
        "        if len(units[i]) < 2:\n",
        "            continue\n",
        "        lab = df[df['OMOPid'] == omops[i]]\n",
        "        # if a max unit exists (frequency greater than 0.55), add to conversion dict\n",
        "        tf, curr_units, maxUnit = getMaxUnit(lab, 0.55)\n",
        "        if tf:\n",
        "            omopToUnit[omops[i]] = [maxUnit, curr_units]\n",
        "\n",
        "    return omopToUnit\n",
        "\n",
        "unit_map = getUnits(clear_ranges)"
      ],
      "id": "4b2942da"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "727fd5ed"
      },
      "outputs": [],
      "source": [
        "# this is the output from getUnits(clear_ranges).\n",
        "# it maps each lab to its most frequent unit (if it made the cutoff. those that did not make the cutoff are considered too ambiguous and disregarded), and to a list of present units\n",
        "# this map will be used to make conversions\n",
        "unit_map"
      ],
      "id": "727fd5ed"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9hFgiCzBbVtq"
      },
      "source": [
        "# 2. Create dict with conversion multipliers for each conversion necessary"
      ],
      "id": "9hFgiCzBbVtq"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "74813cc2"
      },
      "outputs": [],
      "source": [
        "# Due to the inconsistent nature of the data, it was too difficult to automate the process of getting the conversions between two units.\n",
        "# There were too many typos, and mostly ambiguous abbreviations that made it difficult for anything but a human eye with lab test knowledge to discern what different abbreviations likely meant.\n",
        "# thus the conversions were hard coded, all from the unit map above.\n",
        "conversions = {'mg/24 hr': {'mg/d' : 1},\n",
        "               'mg/24 h' : {'mg/d':1},\n",
        "               'ug/24 hr' : {'mcg/24 h' : 1},\n",
        "               'g/24 hr' : {'g/d' : 1},\n",
        "               'g/24 h' : {'g/d' : 1},\n",
        "               'mm/hr' : {'mm/h' : 1},\n",
        "               'mmol/24 hr' : {'mmol/d': 1},\n",
        "               'mmol/24 h' : {'mmol/d' : 1},\n",
        "\n",
        "               'ug feu/ml' : {'mg/l feu': 1},\n",
        "\n",
        "               'million/ul' : {'x10e12/l' : 1},\n",
        "               'x10e6/ul' : {'x10e12/l' : 1},\n",
        "               'x10e6 cells/ul' : {'x10e12/l' : 1},\n",
        "               'x10e6/ul' : {'x10e12/l' : 1},\n",
        "               '10e6/mm3' : {'x10e12/l' : 1},\n",
        "               'm/ul' : {'x10e12/l' : 1},\n",
        "\n",
        "               'mosm/kg' : {\"mmol/kg\" : 1},\n",
        "               'mosmol/kg' : {\"mmol/kg\" : 1},\n",
        "               'meq/l' : {'mmol/l' : 1},\n",
        "               'eu/dl' : {'mg/dl' : 1},\n",
        "               'mm/l' : {'mmol/l' : 1},\n",
        "               'umol/l' : {'nmol/ml' : 1},\n",
        "               \"nmol/ml\" : {'umol/l' : 1},\n",
        "\n",
        "\n",
        "\n",
        "               'ng/ml/hr' : {'ng/ml/h' : 1},\n",
        "               'units/l' : {'u/l' : 1},\n",
        "               'sec' : {'s' : 1},\n",
        "\n",
        "               'mg/g creat' : {'mg/g cr' : 1},\n",
        "               'mg/mg creat' : {'mg/g creat' : 1000},\n",
        "               'ug/mg creat' : {'mg/g creat' : 1},\n",
        "\n",
        "               'pg/ml' : {'ng/ml': 0.001, 'ng/l' : 1},\n",
        "               'ng/ml' : {'mcg/l' : 1, 'ug/l' : 1, 'ug/l rbc' :1 , 'ug/ml' : .001, 'mg/dl' : .0001, 'pg/ml' : 1000},\n",
        "               'ug/ml' : {'mcg/ml' : 1, 'mg/ml' : .001, 'mg/l' : 1, 'mg/dl' : .1},\n",
        "               'ug/dl' : {'mcg/dl' : 1},\n",
        "               'mg/dl (calc)' : {'mg/dl' : 1},\n",
        "               'mg/dl' : {'g/dl' : .001},\n",
        "               'mg/dl adult' : {'mg/dl' : 1},\n",
        "               'mcg/dl' : {'ug/dl' : 1},\n",
        "               'g/dl' : {'g/dl (calc)' : 1},\n",
        "               'mg/l' : {'ng/ml' : 1000},\n",
        "              'ug/l' : {'ng/ml' : 1, 'mcg/l' : 1, 'mcg/dl' : 0.1},\n",
        "               'u/l' : {'mcg/l' : 1},\n",
        "\n",
        "               'mm[hg]' : {'mm hg' : 1},\n",
        "               'mm b hg' : {'mm hg' : 1},\n",
        "\n",
        "\n",
        "              '% (calc)' : {'%' : 1},\n",
        "              'percent' : {'%' : 1},\n",
        "              '% normal' : {\"%\" : 1},\n",
        "              '%activity' : {\"%\" : 1},\n",
        "              '% activity' : {\"%\" : 1},\n",
        "              '%nhp' : {\"%\" : 1},\n",
        "              '% inhibition' : {\"%\" : 1},\n",
        "              '% nhp' : {\"%\" : 1},\n",
        "              '% act' : {\"%\" : 1},\n",
        "              '% of total' : {\"%\" : 1},\n",
        "              '% lymphs' : {\"%\" : 1},\n",
        "\n",
        "\n",
        "               'iu/l' : {'u/l' : 1},\n",
        "               'uiu/ml' : {'miu/l' : 1},\n",
        "               'miu/ml' : {'iu/l' : 1},\n",
        "\n",
        "\n",
        "              'thousand/ul' : {'x10e9/l' : 1},\n",
        "              'x10e3/ul' : {\"x10e9/l\" : 1},\n",
        "              'x10e3 cells/ul' : {\"x10e9/l\" : 1},\n",
        "              'k/ul' : {\"x10e9/l\" : 1, 'x10e3/ul' : 1},\n",
        "              'k/mm3': {\"x10e9/l\" : 1, 'x10e3/ul' : 1},\n",
        "              '10e3/mm3' : {'x10e9/l' : 1},\n",
        "              '10*3/ul' : {'x10e9/l' : 1},\n",
        "              'th/mm3' : {'x10e9/l' : 1},\n",
        "\n",
        "               'cells/ul' : {'x10e6/l' : 1, 'x10e9/l' : 0.001, '/ul' : 1},\n",
        "               'cells/mcl': {'cells/ul' : 1},\n",
        "\n",
        "              '/ul' : {'x10e6/l' : 1}}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        ""
      ],
      "id": "74813cc2"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JrSEl7yicEIK"
      },
      "source": [
        "# 3. Implement convertUnits(df)"
      ],
      "id": "JrSEl7yicEIK"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "60c00f52"
      },
      "outputs": [],
      "source": [
        "from statistics import mode\n",
        "\n",
        "# converts units in a df for a lab to the most frequent unit to ensure all ranges are of same unit\n",
        "# returns a the df with the conversions applied.\n",
        "\n",
        "def convertUnits(df):\n",
        "    units = list(df['unit'])\n",
        "    maxUnit = mode(units)\n",
        "\n",
        "    # df where unit is already most common\n",
        "    keep = df[df['unit'] == maxUnit]\n",
        "\n",
        "    # df where unit can be converted to the most common\n",
        "    mid = df[(df['unit'].isin(conversions)) & (df['unit'] != maxUnit)]\n",
        "\n",
        "    multipliers = []\n",
        "\n",
        "    for unit in list(mid['unit']):\n",
        "      # check if conversion exists\n",
        "        if maxUnit in conversions[unit]:\n",
        "          # if so, append the value for this conversion\n",
        "            multipliers.append(conversions[unit][maxUnit])\n",
        "        # if there is no logical conversion (ex: 'mg/24 hr' to 'mg/dl',\n",
        "        # make multiplier negative to then easily filter out of df\n",
        "        else:\n",
        "            multipliers.append(-1)\n",
        "    # apply changes.\n",
        "    mid['range_low'] *= multipliers\n",
        "    mid['range_high'] *= multipliers\n",
        "    mid['unit'] = maxUnit\n",
        "\n",
        "    # drop rows where range is negative, which will be those in which the multiplier was set to -1 to mark impossible conversions\n",
        "    converted = mid[mid[\"range_high\"] >= 0]\n",
        "\n",
        "\n",
        "    # return df with rows converted to the most frequent unit\n",
        "    return keep.append(converted)"
      ],
      "id": "60c00f52"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W_PVhIu7cjq5"
      },
      "outputs": [],
      "source": [],
      "id": "W_PVhIu7cjq5"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OhJd-FHBch-w"
      },
      "source": [
        "# Convert units functionality is now complete. We will apply it later.\n",
        "# First, lets do more exploration and cleaning..."
      ],
      "id": "OhJd-FHBch-w"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i99I0MCBc2sD"
      },
      "source": [
        "We see many of the units are percentages, so we can consolidate these as well."
      ],
      "id": "i99I0MCBc2sD"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7606f11a",
        "outputId": "2ce27be5-b3b2-47ac-e2f6-b0f0f016b144"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'%',\n",
              " '% (calc)',\n",
              " '% act',\n",
              " '% activity',\n",
              " '% inhibition',\n",
              " '% lymphs',\n",
              " '% nhp',\n",
              " '% normal',\n",
              " '% of total',\n",
              " '%activity',\n",
              " '%nhp',\n",
              " 'vol %',\n",
              " 'vol%',\n",
              " 'vols%'}"
            ]
          },
          "execution_count": 61,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "percent_df2 = clear_ranges[clear_ranges[\"unit\"].str.contains(\"%\")]\n",
        "set(percent_df2['unit'])"
      ],
      "id": "7606f11a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b018edd0"
      },
      "outputs": [],
      "source": [
        "## change all units that contain \"%\" to just \"%\"\n",
        "clear_ranges.loc[clear_ranges[\"unit\"].str.contains(\"%\"), \"unit\"] = \"%\"\n",
        "clear_ranges.replace(\"percent\", \"%\", inplace=True)"
      ],
      "id": "b018edd0"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e1946bd5"
      },
      "outputs": [],
      "source": [
        "# get a smaller subset to work on this algorithm\n",
        "labSubset = clear_ranges[(clear_ranges['OMOPid'] == 3024561) & (clear_ranges['gender'] == 'Female') & (clear_ranges['age'] >= 30) & (clear_ranges['age'] < 40) & (clear_ranges['ethnicity'] == \"Not Hispanic or Latino\") & (clear_ranges['race'] == \"White or Caucasian\") & (clear_ranges['age'] == 34)]\n"
      ],
      "id": "e1946bd5"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3xUL969zdKwx"
      },
      "outputs": [],
      "source": [],
      "id": "3xUL969zdKwx"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-MrCQAYdL4h"
      },
      "source": [
        "Remember, the end-goal involves being able to compare patient results to these ranges to determine whether the result is normal. How could we distinguish between ranges 3.5-4.8 vs 3.4-4.7?\n",
        "\n",
        "Here we see there is a need for a confidence interval. We will implement one to merge similar ranges for a lab that have matching demographics."
      ],
      "id": "A-MrCQAYdL4h"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "c680ecd6",
        "outputId": "c1b86da1-b9e1-4eba-980f-ec74b94607df",
        "scrolled": false
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "differing ranges for same omop and demographics.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-2448b9ff-c32f-4c0a-921d-0b568222b705\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB5389</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.5</td>\n",
              "      <td>4.8</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>569264</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB45</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.4</td>\n",
              "      <td>4.7</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2039524</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB6170</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2195584</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB3206</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>3600.0</td>\n",
              "      <td>4800.0</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2230774</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB6170</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.8</td>\n",
              "      <td>4.8</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5248626</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB6170</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.6</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12461477</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB3206</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>3503.0</td>\n",
              "      <td>5000.0</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2448b9ff-c32f-4c0a-921d-0b568222b705')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2448b9ff-c32f-4c0a-921d-0b568222b705 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2448b9ff-c32f-4c0a-921d-0b568222b705');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer   unit  range_low  range_high  gender  \\\n",
              "10        3024561      LAB5389   g/dl        3.5         4.8  Female   \n",
              "569264    3024561        LAB45   g/dl        3.4         4.7  Female   \n",
              "2039524   3024561      LAB6170   g/dl        3.5         5.0  Female   \n",
              "2195584   3024561      LAB3206  mg/dl     3600.0      4800.0  Female   \n",
              "2230774   3024561      LAB6170   g/dl        3.8         4.8  Female   \n",
              "5248626   3024561      LAB6170   g/dl        3.6         5.1  Female   \n",
              "12461477  3024561      LAB3206  mg/dl     3503.0      5000.0  Female   \n",
              "\n",
              "                       ethnicity                race  age LOINCid  \n",
              "10        Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "569264    Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "2039524   Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "2195584   Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "2230774   Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "5248626   Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "12461477  Not Hispanic or Latino  White or Caucasian   34  1751-7  "
            ]
          },
          "execution_count": 64,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# shows need for confidence interval (DONE)\n",
        "print(\"differing ranges for same omop and demographics.\")\n",
        "labSubset.drop_duplicates(subset = ['range_low', 'range_high'])"
      ],
      "id": "c680ecd6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HbyeaQKFd16E"
      },
      "source": [
        "Here we have another example, but even with **the same manufacturer**. This emphasizes how inconsistent the EHR system is, but as far as it concerns our study, it strongly emphasizes the need for a confidence interval to merge these similar ranges."
      ],
      "id": "HbyeaQKFd16E"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "01f0ed76",
        "outputId": "16241fde-e32b-49c8-d7ce-7304bc421b57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Example of discordant ranges for rows with same lab (OMOPid), manufacturer, and demographics\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e525ef1d-2f95-4563-bbab-c0a68bddd8e1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1101</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB45</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.5</td>\n",
              "      <td>4.8</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>569264</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB45</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.4</td>\n",
              "      <td>4.7</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3388387</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB45</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8317838</th>\n",
              "      <td>3024561</td>\n",
              "      <td>LAB45</td>\n",
              "      <td>g/dl</td>\n",
              "      <td>3.6</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>34</td>\n",
              "      <td>1751-7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e525ef1d-2f95-4563-bbab-c0a68bddd8e1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e525ef1d-2f95-4563-bbab-c0a68bddd8e1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e525ef1d-2f95-4563-bbab-c0a68bddd8e1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          OMOPid manufacturer  unit  range_low  range_high  gender  \\\n",
              "1101     3024561        LAB45  g/dl        3.5         4.8  Female   \n",
              "569264   3024561        LAB45  g/dl        3.4         4.7  Female   \n",
              "3388387  3024561        LAB45  g/dl        3.5         5.0  Female   \n",
              "8317838  3024561        LAB45  g/dl        3.6         5.1  Female   \n",
              "\n",
              "                      ethnicity                race  age LOINCid  \n",
              "1101     Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "569264   Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "3388387  Not Hispanic or Latino  White or Caucasian   34  1751-7  \n",
              "8317838  Not Hispanic or Latino  White or Caucasian   34  1751-7  "
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Another example, but even with same manufacturer. Emphasizes need for confidence interval (DONE)\n",
        "print('Example of discordant ranges for rows with same lab (OMOPid), manufacturer, and demographics')\n",
        "labSubset[labSubset['manufacturer'] == 'LAB45'].drop_duplicates(subset = ['range_low', 'range_high'])"
      ],
      "id": "01f0ed76"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1ca6e16"
      },
      "source": [
        "# Ok, so in order to do further consolidation. We need to:\n",
        "\n",
        "  a) **merge similar ranges** using a confidence interval\n",
        "\n",
        "\n",
        "    merge_similar_ranges(df, ci)\n",
        "\n",
        "  b) **merge demographics**. That is, set demographic values to 'any' in places where, for example, the range is the same regardless of gender. In this case the gender column should say 'any'.\n",
        "\n",
        "\n",
        "    merge_demographics(df)"
      ],
      "id": "c1ca6e16"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "7771ef1f",
        "outputId": "3fe7c29d-f791-4bf3-99f4-1e4c17e6118c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b2a76fde-670f-4dbd-a8db-399fcc084f0a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>3006576</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mmol/l</td>\n",
              "      <td>22.00</td>\n",
              "      <td>27.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>1959-6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "      <td>2160-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "      <td>6690-2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "      <td>43743-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "      <td>731-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "      <td>25132-2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13872056 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b2a76fde-670f-4dbd-a8db-399fcc084f0a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b2a76fde-670f-4dbd-a8db-399fcc084f0a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b2a76fde-670f-4dbd-a8db-399fcc084f0a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer            unit  range_low  range_high  gender  \\\n",
              "1         3013502      LAB5808               %      95.00       99.00  Female   \n",
              "2         3013502      LAB5809               %      95.00       99.00  Female   \n",
              "3         3027315      LAB5808           mm hg      80.00      100.00  Female   \n",
              "4         3027315      LAB5809           mm hg      80.00      100.00  Female   \n",
              "5         3006576      LAB5808          mmol/l      22.00       27.00  Female   \n",
              "...           ...          ...             ...        ...         ...     ...   \n",
              "14244781  3016723      LAB3001           mg/dl       0.61        1.24    Male   \n",
              "14244782  3000905       LAB294         x10e9/l       3.40       10.00  Female   \n",
              "14244783  3046553       LAB293         x10e9/l       0.00        0.20  Female   \n",
              "14244784  3004327       LAB293         x10e9/l       0.90        2.90    Male   \n",
              "14244785  3023996       LC3743  mmol/mol creat       1.00       14.00    Male   \n",
              "\n",
              "                       ethnicity                race  age  LOINCid  \n",
              "1         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "2         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "3         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "4         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "5         Not Hispanic or Latino               Asian    0   1959-6  \n",
              "...                          ...                 ...  ...      ...  \n",
              "14244781      Hispanic or Latino  White or Caucasian   53   2160-0  \n",
              "14244782                 Unknown             Unknown   71   6690-2  \n",
              "14244783      Hispanic or Latino  White or Caucasian    2  43743-4  \n",
              "14244784  Not Hispanic or Latino  White or Caucasian   36    731-0  \n",
              "14244785      Hispanic or Latino             Unknown   13  25132-2  \n",
              "\n",
              "[13872056 rows x 10 columns]"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "clear_ranges"
      ],
      "id": "7771ef1f"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "29b75aad"
      },
      "outputs": [],
      "source": [
        "df = clear_ranges"
      ],
      "id": "29b75aad"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "41ceb285"
      },
      "outputs": [],
      "source": [
        "#define our own age ranges\n",
        "ages = [[0,10], [10,20], [20,30], [30,40], [40,50], [50,60], [60,70], [70,80], [80,90], [90,100]]\n",
        "\n",
        "# create list for each possible demographic\n",
        "races = list(set(df['race']))\n",
        "races.remove('Unknown')\n",
        "\n",
        "ethnicities = list(set(df['ethnicity']))\n",
        "ethnicities.remove('Unknown')\n",
        "\n",
        "genders = list(set(df['gender']))"
      ],
      "id": "41ceb285"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f3d092dd",
        "outputId": "942ee976-cc18-499e-b973-3713fab7d7d3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Hispanic or Latino', 'Not Hispanic or Latino']"
            ]
          },
          "execution_count": 70,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ethnicities"
      ],
      "id": "f3d092dd"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aYzKFxaLhCos"
      },
      "source": [
        "# A) Merge similar ranges"
      ],
      "id": "aYzKFxaLhCos"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2c03818a"
      },
      "outputs": [],
      "source": [
        "def merge_similar_ranges(df, ci = float):\n",
        "  '''Merges similar ranges for a given lab (df), given a confidence interval (ci), in place'''\n",
        "    vis = []\n",
        "    range_CI = {}\n",
        "    # remember to consider that one range might be mapped to another, which is mapped to a different one. Might have to merge all in this case.\n",
        "    # To overcome this obstacle, get the mappings between ranges that are similar enough to merge into the same range\n",
        "    for i, row in df.iterrows():\n",
        "        index = row\n",
        "        low = row['range_low']\n",
        "        high = row['range_high']\n",
        "\n",
        "        curr_interval = str(low)+'+'+str(high)\n",
        "\n",
        "        # if we saw this range already, continue. If we haven't, go through and see if it falls in the CI of any visited range\n",
        "        if [low, high] in vis:\n",
        "            new_range = range_CI[curr_interval]\n",
        "            df.at[i,'range_low'] = new_range[0]\n",
        "            df.at[i,'range_high'] = new_range[1]\n",
        "            continue\n",
        "\n",
        "        # initially, map this range to itself in case it doesn't end up in the CI of another range.\n",
        "        range_CI[curr_interval] = [low, high]\n",
        "\n",
        "        # get the lower and upper bound intervals for the low and high values of the range\n",
        "        low_int = [low-low*ci, low+low*ci]\n",
        "        high_int = [high-high*ci, high+high*ci]\n",
        "\n",
        "        # iterate through unique ranges we already visited\n",
        "        for interval in vis:\n",
        "            # if this range falls in the CI of one of the ranges we already saw\n",
        "            if (interval[0] >= low_int[0]) and (interval[0] <= low_int[1]) and (interval[1] >= high_int[0]) and (interval[1] <= high_int[1]):\n",
        "\n",
        "                #map the current range to this one\n",
        "                range_CI[curr_interval] = [interval[0],interval[1]]\n",
        "\n",
        "                # change the value of this range to match the other\n",
        "                df.at[i,'range_low'] = interval[0]\n",
        "                df.at[i,'range_high'] = interval[1]\n",
        "            break\n",
        "\n",
        "        if [low,high] not in vis:\n",
        "            vis.append((low,high))\n",
        "\n",
        "    return\n",
        ""
      ],
      "id": "2c03818a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MqCNEEgnitqt"
      },
      "source": [
        "# Now lets test merge_similar_ranges() on a sample lab and check if it works."
      ],
      "id": "MqCNEEgnitqt"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4947f634"
      },
      "outputs": [],
      "source": [
        "# original lab sample, good for testing code below:\n",
        "# to test with pdfs:\n",
        "sample_lab = df[df['LOINCid'] == '789-8']\n",
        "x = sample_lab"
      ],
      "id": "4947f634"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "75f1621b",
        "outputId": "05a8d7c7-ea17-4c63-c336-11cfc3c30335"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-5bf50bc8-6d2f-4bd8-9c87-9224c5fe0a38\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>35</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>35</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>35</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>35</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>35</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244533</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>38</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244550</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.4</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>33</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244669</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.4</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>22</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244731</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.2</td>\n",
              "      <td>5.6</td>\n",
              "      <td>Male</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>16</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244778</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>70</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>787184 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5bf50bc8-6d2f-4bd8-9c87-9224c5fe0a38')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5bf50bc8-6d2f-4bd8-9c87-9224c5fe0a38 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5bf50bc8-6d2f-4bd8-9c87-9224c5fe0a38');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer      unit  range_low  range_high  gender  \\\n",
              "90        3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "91        3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "92        3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "93        3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "94        3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "...           ...          ...       ...        ...         ...     ...   \n",
              "14244533  3020416       LAB294  x10e12/l        4.0         5.2  Female   \n",
              "14244550  3020416       LAB294  x10e12/l        4.4         5.9    Male   \n",
              "14244669  3020416       LAB293  x10e12/l        4.4         5.9    Male   \n",
              "14244731  3020416       LAB293  x10e12/l        4.2         5.6    Male   \n",
              "14244778  3020416       LAB293  x10e12/l        4.0         5.2  Female   \n",
              "\n",
              "                       ethnicity                race  age LOINCid  \n",
              "90        Not Hispanic or Latino  White or Caucasian   35   789-8  \n",
              "91        Not Hispanic or Latino  White or Caucasian   35   789-8  \n",
              "92        Not Hispanic or Latino  White or Caucasian   35   789-8  \n",
              "93        Not Hispanic or Latino  White or Caucasian   35   789-8  \n",
              "94        Not Hispanic or Latino  White or Caucasian   35   789-8  \n",
              "...                          ...                 ...  ...     ...  \n",
              "14244533  Not Hispanic or Latino             Unknown   38   789-8  \n",
              "14244550  Not Hispanic or Latino  White or Caucasian   33   789-8  \n",
              "14244669  Not Hispanic or Latino               Asian   22   789-8  \n",
              "14244731                 Unknown  White or Caucasian   16   789-8  \n",
              "14244778                 Unknown  White or Caucasian   70   789-8  \n",
              "\n",
              "[787184 rows x 10 columns]"
            ]
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "x"
      ],
      "id": "75f1621b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "16a5869d"
      },
      "outputs": [],
      "source": [
        "sample1 = x[x['range_low'] == 4].sample(50)\n",
        "sample2 = x[x['range_low'] == 4.14].sample(50)\n",
        "sample3 = x[x['range_low'] == 3.8]"
      ],
      "id": "16a5869d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "69729acb",
        "outputId": "8aa2ab36-0d2a-458a-ac32-19cd7997add4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-76-de6316c891f3>:2: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
            "  z = sample1.append(sample2).append(sample3)\n",
            "<ipython-input-76-de6316c891f3>:2: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
            "  z = sample1.append(sample2).append(sample3)\n"
          ]
        }
      ],
      "source": [
        "# get a sample of data for each range from a lab\n",
        "z = sample1.append(sample2).append(sample3)"
      ],
      "id": "69729acb"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "dc79ce01",
        "outputId": "f7fdea17-60fb-4b3d-83d9-f8d44bfe93cc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-da958f2a-45da-4b90-a82d-a52b87c1c5d0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12425516</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>33</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9010898</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>44</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11036010</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>42</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6832396</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Native Hawaiian or Other Pacific Islander</td>\n",
              "      <td>55</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2069836</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>57</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14238589</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>3.8</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>26</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14240514</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>3.8</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>54</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14243171</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>3.8</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>38</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244120</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>3.8</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>59</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244444</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>3.8</td>\n",
              "      <td>5.1</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>19</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13373 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-da958f2a-45da-4b90-a82d-a52b87c1c5d0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-da958f2a-45da-4b90-a82d-a52b87c1c5d0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-da958f2a-45da-4b90-a82d-a52b87c1c5d0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer        unit  range_low  range_high  gender  \\\n",
              "12425516  3020416       LAB293    x10e12/l        4.0         5.2  Female   \n",
              "9010898   3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "11036010  3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "6832396   3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "2069836   3020416       LAB293    x10e12/l        4.0         5.2  Female   \n",
              "...           ...          ...         ...        ...         ...     ...   \n",
              "14238589  3020416       LAB293  million/ul        3.8         5.1  Female   \n",
              "14240514  3020416       LAB293  million/ul        3.8         5.1  Female   \n",
              "14243171  3020416       LAB293  million/ul        3.8         5.1  Female   \n",
              "14244120  3020416       LAB293  million/ul        3.8         5.1  Female   \n",
              "14244444  3020416       LAB294  million/ul        3.8         5.1  Female   \n",
              "\n",
              "                       ethnicity                                       race  \\\n",
              "12425516  Not Hispanic or Latino                  Black or African American   \n",
              "9010898                  Unknown                                    Unknown   \n",
              "11036010  Not Hispanic or Latino                         White or Caucasian   \n",
              "6832396   Not Hispanic or Latino  Native Hawaiian or Other Pacific Islander   \n",
              "2069836   Not Hispanic or Latino                         White or Caucasian   \n",
              "...                          ...                                        ...   \n",
              "14238589  Not Hispanic or Latino                         White or Caucasian   \n",
              "14240514  Not Hispanic or Latino                         White or Caucasian   \n",
              "14243171      Hispanic or Latino                                    Unknown   \n",
              "14244120  Not Hispanic or Latino                         White or Caucasian   \n",
              "14244444  Not Hispanic or Latino                         White or Caucasian   \n",
              "\n",
              "          age LOINCid  \n",
              "12425516   33   789-8  \n",
              "9010898    44   789-8  \n",
              "11036010   42   789-8  \n",
              "6832396    55   789-8  \n",
              "2069836    57   789-8  \n",
              "...       ...     ...  \n",
              "14238589   26   789-8  \n",
              "14240514   54   789-8  \n",
              "14243171   38   789-8  \n",
              "14244120   59   789-8  \n",
              "14244444   19   789-8  \n",
              "\n",
              "[13373 rows x 10 columns]"
            ]
          },
          "execution_count": 77,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "z"
      ],
      "id": "dc79ce01"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cb5ebcd6"
      },
      "outputs": [],
      "source": [
        "merge_similar_ranges(z, ci = 0.1)"
      ],
      "id": "cb5ebcd6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jYuo_cMJkB7f"
      },
      "source": [
        "### compare z before and after merge_similar_ranges()\n",
        "#### note that the ci = 0.1, so when taking 4.0 as the low range, it looks for anything between 3.6 and 4.4, and merges the range to 4.0.\n",
        "### Success!"
      ],
      "id": "jYuo_cMJkB7f"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "2fa7823e",
        "outputId": "316690da-2291-4e60-c779-8df8fd17b407",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e27a9011-0874-4421-8506-f2ac25152f31\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12425516</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>33</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9010898</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>44</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11036010</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>42</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6832396</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Native Hawaiian or Other Pacific Islander</td>\n",
              "      <td>55</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2069836</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>57</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14238589</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>26</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14240514</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>54</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14243171</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>38</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244120</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>59</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244444</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>19</td>\n",
              "      <td>789-8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13373 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e27a9011-0874-4421-8506-f2ac25152f31')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e27a9011-0874-4421-8506-f2ac25152f31 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e27a9011-0874-4421-8506-f2ac25152f31');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer        unit  range_low  range_high  gender  \\\n",
              "12425516  3020416       LAB293    x10e12/l        4.0         5.2  Female   \n",
              "9010898   3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "11036010  3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "6832396   3020416       LAB294    x10e12/l        4.0         5.2  Female   \n",
              "2069836   3020416       LAB293    x10e12/l        4.0         5.2  Female   \n",
              "...           ...          ...         ...        ...         ...     ...   \n",
              "14238589  3020416       LAB293  million/ul        4.0         5.2  Female   \n",
              "14240514  3020416       LAB293  million/ul        4.0         5.2  Female   \n",
              "14243171  3020416       LAB293  million/ul        4.0         5.2  Female   \n",
              "14244120  3020416       LAB293  million/ul        4.0         5.2  Female   \n",
              "14244444  3020416       LAB294  million/ul        4.0         5.2  Female   \n",
              "\n",
              "                       ethnicity                                       race  \\\n",
              "12425516  Not Hispanic or Latino                  Black or African American   \n",
              "9010898                  Unknown                                    Unknown   \n",
              "11036010  Not Hispanic or Latino                         White or Caucasian   \n",
              "6832396   Not Hispanic or Latino  Native Hawaiian or Other Pacific Islander   \n",
              "2069836   Not Hispanic or Latino                         White or Caucasian   \n",
              "...                          ...                                        ...   \n",
              "14238589  Not Hispanic or Latino                         White or Caucasian   \n",
              "14240514  Not Hispanic or Latino                         White or Caucasian   \n",
              "14243171      Hispanic or Latino                                    Unknown   \n",
              "14244120  Not Hispanic or Latino                         White or Caucasian   \n",
              "14244444  Not Hispanic or Latino                         White or Caucasian   \n",
              "\n",
              "          age LOINCid  \n",
              "12425516   33   789-8  \n",
              "9010898    44   789-8  \n",
              "11036010   42   789-8  \n",
              "6832396    55   789-8  \n",
              "2069836    57   789-8  \n",
              "...       ...     ...  \n",
              "14238589   26   789-8  \n",
              "14240514   54   789-8  \n",
              "14243171   38   789-8  \n",
              "14244120   59   789-8  \n",
              "14244444   19   789-8  \n",
              "\n",
              "[13373 rows x 10 columns]"
            ]
          },
          "execution_count": 79,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "z"
      ],
      "id": "2fa7823e"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NG_BcDkgk_C3"
      },
      "source": [
        "# B) Merge Demographics"
      ],
      "id": "NG_BcDkgk_C3"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "db8b9368"
      },
      "outputs": [],
      "source": [
        "# Input a list of ages. Returns True if 50% of age bins are present (bin_size=10 would give 0-10, 10-20, ..., 90-100),\n",
        "# indicating that we can replace the age column with 'any'\n",
        "def all_ages(ages, bin_size = int):\n",
        "    # loop through all age bins (0-10, 10-20, ..., 90-100)\n",
        "    vals = []\n",
        "    for i in range(0,100,bin_size):\n",
        "        lower = i\n",
        "        upper = i+bin_size\n",
        "        # append True if there's an age in ages list that falls in the current bin\n",
        "        vals.append(any([True for age in ages if age >= lower and age < upper]))\n",
        "\n",
        "    if (sum(vals)/len(vals)) >= 0.50:\n",
        "        return True\n",
        "\n",
        "    return False\n"
      ],
      "id": "db8b9368"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ab096107",
        "outputId": "80220f82-53cd-45ba-e3bf-e1c6740da7e0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Hispanic or Latino', 'Not Hispanic or Latino']"
            ]
          },
          "execution_count": 81,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ethnicities"
      ],
      "id": "ab096107"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "52c3d700",
        "outputId": "7af755f2-f491-4d6c-f7cd-87b5bac59cbb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['White or Caucasian',\n",
              " 'American Indian or Alaska Native',\n",
              " 'Native Hawaiian or Other Pacific Islander',\n",
              " 'Asian',\n",
              " 'Black or African American']"
            ]
          },
          "execution_count": 82,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "races"
      ],
      "id": "52c3d700"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "94258519"
      },
      "outputs": [],
      "source": [
        "# Inputs a df for a lab with ranges already merged where necessary. Returns a df with 'any'\n",
        "# in the demographic columns if the range does not depend on this demographic (determined on whether all possible\n",
        "# values for a demographic are present for a given range). not in place.\n",
        "def merge_demographics(df):\n",
        "\n",
        "    # initialize df to append to during iteration\n",
        "    output_df = pd.DataFrame(columns=list(df.columns))\n",
        "\n",
        "    # break into new dfs per range, search to see if value can be replaced by 'any' for each range\n",
        "    ranges = df[['range_low', 'range_high']].drop_duplicates()\n",
        "\n",
        "\n",
        "    # iterate through unique ranges for this lab\n",
        "    for i, lab_range in ranges.iterrows():\n",
        "        low = lab_range['range_low']\n",
        "        high = lab_range['range_high']\n",
        "\n",
        "        # get all rows that have this specific range for this lab\n",
        "        curr_df = df[(df['range_low'] == low) & (df['range_high'] == high)]\n",
        "\n",
        "\n",
        "        # now check if the df for this range contains all values for each demographic and if so, change to 'any'\n",
        "        # then append all rows to the new table\n",
        "\n",
        "        # check if all values for gender are present\n",
        "        curr_gender_vals = list(set(curr_df['gender']))\n",
        "        if len(genders) == len(curr_gender_vals):\n",
        "            curr_df['gender'] = 'any'\n",
        "\n",
        "        # check if all values for ethnicity are present\n",
        "        curr_ethnicity_vals = list(set(curr_df['ethnicity']))\n",
        "        # value could be \"Unknown\". check if all defined ethnicities are in the current range\n",
        "        if all([x in curr_ethnicity_vals for x in ethnicities]):\n",
        "            curr_df['ethnicity'] = 'any'\n",
        "\n",
        "        # check if all values for race are present\n",
        "        curr_race_vals = list(set(curr_df['race']))\n",
        "        # value could be \"Unknown\". check if all defined races are in the current range\n",
        "        if all([x in curr_race_vals for x in races]):\n",
        "            curr_df['race'] = 'any'\n",
        "\n",
        "        # get age in intervals:\n",
        "        curr_df['age_range'] = [ages[age//10] for age in curr_df['age']]\n",
        "\n",
        "        # get first digit of age range to drop duplicates on later (can't drop duplicates on age_range because it's a list)\n",
        "        curr_df['age_index'] = [age_range[0] for age_range in curr_df['age_range']]\n",
        "\n",
        "        # check if all values for age (range) are present\n",
        "        if all_ages(list(set(curr_df['age'])), bin_size = 30):\n",
        "            curr_df['age'] = 'any'\n",
        "\n",
        "        # drop duplicates\n",
        "        curr_df.drop_duplicates(subset = (['gender', 'ethnicity', 'race', 'age_index']), inplace = True)\n",
        "        # no longer need age_index col, drop it\n",
        "        curr_df.drop('age_index', inplace=True, axis = 1)\n",
        "\n",
        "\n",
        "        #concatenate result to output\n",
        "        output_df = output_df.append(curr_df)\n",
        "\n",
        "    return output_df\n"
      ],
      "id": "94258519"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wjv1bt_2nc7x"
      },
      "source": [
        "# Lets try merge_demographics() on our sample dataset..."
      ],
      "id": "wjv1bt_2nc7x"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3c01b09b"
      },
      "outputs": [],
      "source": [
        "# merge_demographics() on sample data set from before: z\n",
        "output = merge_demographics(z)"
      ],
      "id": "3c01b09b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "760c15a0",
        "outputId": "3085e455-7502-4154-ef18-bbf06d4cd0ce",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e29d098b-f7af-49d2-a6b3-2d55617cac3f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "      <th>age_range</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12425516</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[30, 40]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9010898</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[40, 50]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6832396</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4088755</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[60, 70]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12914164</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[20, 30]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1349612</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[70, 80]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4196126</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[10, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13331515</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[0, 10]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5739968</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[80, 90]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2384990</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>million/ul</td>\n",
              "      <td>4.00</td>\n",
              "      <td>5.2</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[90, 100]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12709954</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e12/l</td>\n",
              "      <td>4.00</td>\n",
              "      <td>6.6</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>0</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[0, 10]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5148665</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[60, 70]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9476129</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Asian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7996663</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8342842</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[70, 80]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14051799</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8120200</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3493123</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[20, 30]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7684824</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Asian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[60, 70]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8328433</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[80, 90]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10851506</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[10, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12949315</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Asian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[20, 30]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3334528</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[20, 30]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12675998</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Asian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[10, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9589599</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[70, 80]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8262285</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Asian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[70, 80]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12240673</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[60, 70]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>344000</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[30, 40]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13889781</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[40, 50]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2504522</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[40, 50]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9655921</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e6/ul</td>\n",
              "      <td>4.14</td>\n",
              "      <td>5.8</td>\n",
              "      <td>Male</td>\n",
              "      <td>any</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[30, 40]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1894956</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>10*6/ul</td>\n",
              "      <td>3.80</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[70, 80]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8884081</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>10*6/ul</td>\n",
              "      <td>3.80</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[20, 30]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9888944</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB11007</td>\n",
              "      <td>10*6/ul</td>\n",
              "      <td>3.80</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[50, 60]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11325452</th>\n",
              "      <td>3020416</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>10*6/ul</td>\n",
              "      <td>3.80</td>\n",
              "      <td>5.9</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>any</td>\n",
              "      <td>789-8</td>\n",
              "      <td>[80, 90]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e29d098b-f7af-49d2-a6b3-2d55617cac3f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e29d098b-f7af-49d2-a6b3-2d55617cac3f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e29d098b-f7af-49d2-a6b3-2d55617cac3f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer        unit  range_low  range_high  gender  \\\n",
              "12425516  3020416       LAB293    x10e12/l       4.00         5.2     any   \n",
              "9010898   3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "6832396   3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "4088755   3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "12914164  3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "1349612   3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "4196126   3020416       LAB293    x10e12/l       4.00         5.2     any   \n",
              "13331515  3020416       LAB293    x10e12/l       4.00         5.2     any   \n",
              "5739968   3020416       LAB294    x10e12/l       4.00         5.2     any   \n",
              "2384990   3020416       LAB293  million/ul       4.00         5.2     any   \n",
              "12709954  3020416       LAB294    x10e12/l       4.00         6.6  Female   \n",
              "5148665   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "9476129   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "7996663   3020416       LAB294    x10e6/ul       4.14         5.8    Male   \n",
              "8342842   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "14051799  3020416       LAB294    x10e6/ul       4.14         5.8    Male   \n",
              "8120200   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "3493123   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "7684824   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "8328433   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "10851506  3020416       LAB294    x10e6/ul       4.14         5.8    Male   \n",
              "12949315  3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "3334528   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "12675998  3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "9589599   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "8262285   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "12240673  3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "344000    3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "13889781  3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "2504522   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "9655921   3020416       LAB293    x10e6/ul       4.14         5.8    Male   \n",
              "1894956   3020416       LAB293     10*6/ul       3.80         5.9    Male   \n",
              "8884081   3020416       LAB293     10*6/ul       3.80         5.9    Male   \n",
              "9888944   3020416     LAB11007     10*6/ul       3.80         5.9    Male   \n",
              "11325452  3020416       LAB293     10*6/ul       3.80         5.9    Male   \n",
              "\n",
              "                       ethnicity                       race  age LOINCid  \\\n",
              "12425516                     any                        any  any   789-8   \n",
              "9010898                      any                        any  any   789-8   \n",
              "6832396                      any                        any  any   789-8   \n",
              "4088755                      any                        any  any   789-8   \n",
              "12914164                     any                        any  any   789-8   \n",
              "1349612                      any                        any  any   789-8   \n",
              "4196126                      any                        any  any   789-8   \n",
              "13331515                     any                        any  any   789-8   \n",
              "5739968                      any                        any  any   789-8   \n",
              "2384990                      any                        any  any   789-8   \n",
              "12709954                 Unknown                    Unknown    0   789-8   \n",
              "5148665                      any         White or Caucasian  any   789-8   \n",
              "9476129                      any                      Asian  any   789-8   \n",
              "7996663                      any  Black or African American  any   789-8   \n",
              "8342842                      any         White or Caucasian  any   789-8   \n",
              "14051799                     any         White or Caucasian  any   789-8   \n",
              "8120200                      any                    Unknown  any   789-8   \n",
              "3493123                      any         White or Caucasian  any   789-8   \n",
              "7684824                      any                      Asian  any   789-8   \n",
              "8328433                      any         White or Caucasian  any   789-8   \n",
              "10851506                     any                    Unknown  any   789-8   \n",
              "12949315                     any                      Asian  any   789-8   \n",
              "3334528                      any                    Unknown  any   789-8   \n",
              "12675998                     any                      Asian  any   789-8   \n",
              "9589599                      any                    Unknown  any   789-8   \n",
              "8262285                      any                      Asian  any   789-8   \n",
              "12240673                     any                    Unknown  any   789-8   \n",
              "344000                       any                    Unknown  any   789-8   \n",
              "13889781                     any  Black or African American  any   789-8   \n",
              "2504522                      any                    Unknown  any   789-8   \n",
              "9655921                      any  Black or African American  any   789-8   \n",
              "1894956   Not Hispanic or Latino         White or Caucasian  any   789-8   \n",
              "8884081                  Unknown                    Unknown  any   789-8   \n",
              "9888944   Not Hispanic or Latino         White or Caucasian  any   789-8   \n",
              "11325452  Not Hispanic or Latino         White or Caucasian  any   789-8   \n",
              "\n",
              "          age_range  \n",
              "12425516   [30, 40]  \n",
              "9010898    [40, 50]  \n",
              "6832396    [50, 60]  \n",
              "4088755    [60, 70]  \n",
              "12914164   [20, 30]  \n",
              "1349612    [70, 80]  \n",
              "4196126    [10, 20]  \n",
              "13331515    [0, 10]  \n",
              "5739968    [80, 90]  \n",
              "2384990   [90, 100]  \n",
              "12709954    [0, 10]  \n",
              "5148665    [60, 70]  \n",
              "9476129    [50, 60]  \n",
              "7996663    [50, 60]  \n",
              "8342842    [70, 80]  \n",
              "14051799   [50, 60]  \n",
              "8120200    [50, 60]  \n",
              "3493123    [20, 30]  \n",
              "7684824    [60, 70]  \n",
              "8328433    [80, 90]  \n",
              "10851506   [10, 20]  \n",
              "12949315   [20, 30]  \n",
              "3334528    [20, 30]  \n",
              "12675998   [10, 20]  \n",
              "9589599    [70, 80]  \n",
              "8262285    [70, 80]  \n",
              "12240673   [60, 70]  \n",
              "344000     [30, 40]  \n",
              "13889781   [40, 50]  \n",
              "2504522    [40, 50]  \n",
              "9655921    [30, 40]  \n",
              "1894956    [70, 80]  \n",
              "8884081    [20, 30]  \n",
              "9888944    [50, 60]  \n",
              "11325452   [80, 90]  "
            ]
          },
          "execution_count": 85,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# print the output, which is basically the concatenated dfs for each range with demographic set to 'any' where it meets the condition\n",
        "output"
      ],
      "id": "760c15a0"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lclLW27oo61m"
      },
      "outputs": [],
      "source": [],
      "id": "lclLW27oo61m"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXOsiE7SpBwd"
      },
      "source": [
        "# Great. With these methods the lab ranges can be used as reference for individual patient results. Now lets run it on all the data.\n",
        "\n",
        "We will use clear_ranges as input, the following code will\n",
        "- loop through each lab (OMOPid)\n",
        "  - Check if there is only one unit\n",
        "    - if so:\n",
        "      - merges similar ranges\n",
        "      - merges demographics\n",
        "      - appends to the output\n",
        "      - continues\n",
        "    - if not:\n",
        "      - checks if the units in this lab are convertible\n",
        "      - if not, continues\n",
        "  - if its made it this far, it means we have multiple units and they are convertible. proceeds to:\n",
        "    - convert units\n",
        "    - merge similar ranges\n",
        "    - merge demographics\n",
        "    - append to the output"
      ],
      "id": "mXOsiE7SpBwd"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "80d6ecd1",
        "outputId": "00568d6f-dae1-45d1-b7a2-baa3cd639900"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-05051b2c-3161-4078-94c4-d17cfe6eebe3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3013502</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>%</td>\n",
              "      <td>95.00</td>\n",
              "      <td>99.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>20564-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3027315</td>\n",
              "      <td>LAB5809</td>\n",
              "      <td>mm hg</td>\n",
              "      <td>80.00</td>\n",
              "      <td>100.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>11556-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>3006576</td>\n",
              "      <td>LAB5808</td>\n",
              "      <td>mmol/l</td>\n",
              "      <td>22.00</td>\n",
              "      <td>27.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>0</td>\n",
              "      <td>1959-6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244781</th>\n",
              "      <td>3016723</td>\n",
              "      <td>LAB3001</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.24</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>53</td>\n",
              "      <td>2160-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244782</th>\n",
              "      <td>3000905</td>\n",
              "      <td>LAB294</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>3.40</td>\n",
              "      <td>10.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>71</td>\n",
              "      <td>6690-2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244783</th>\n",
              "      <td>3046553</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>2</td>\n",
              "      <td>43743-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244784</th>\n",
              "      <td>3004327</td>\n",
              "      <td>LAB293</td>\n",
              "      <td>x10e9/l</td>\n",
              "      <td>0.90</td>\n",
              "      <td>2.90</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>36</td>\n",
              "      <td>731-0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244785</th>\n",
              "      <td>3023996</td>\n",
              "      <td>LC3743</td>\n",
              "      <td>mmol/mol creat</td>\n",
              "      <td>1.00</td>\n",
              "      <td>14.00</td>\n",
              "      <td>Male</td>\n",
              "      <td>Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>13</td>\n",
              "      <td>25132-2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13872056 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-05051b2c-3161-4078-94c4-d17cfe6eebe3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-05051b2c-3161-4078-94c4-d17cfe6eebe3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-05051b2c-3161-4078-94c4-d17cfe6eebe3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "           OMOPid manufacturer            unit  range_low  range_high  gender  \\\n",
              "1         3013502      LAB5808               %      95.00       99.00  Female   \n",
              "2         3013502      LAB5809               %      95.00       99.00  Female   \n",
              "3         3027315      LAB5808           mm hg      80.00      100.00  Female   \n",
              "4         3027315      LAB5809           mm hg      80.00      100.00  Female   \n",
              "5         3006576      LAB5808          mmol/l      22.00       27.00  Female   \n",
              "...           ...          ...             ...        ...         ...     ...   \n",
              "14244781  3016723      LAB3001           mg/dl       0.61        1.24    Male   \n",
              "14244782  3000905       LAB294         x10e9/l       3.40       10.00  Female   \n",
              "14244783  3046553       LAB293         x10e9/l       0.00        0.20  Female   \n",
              "14244784  3004327       LAB293         x10e9/l       0.90        2.90    Male   \n",
              "14244785  3023996       LC3743  mmol/mol creat       1.00       14.00    Male   \n",
              "\n",
              "                       ethnicity                race  age  LOINCid  \n",
              "1         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "2         Not Hispanic or Latino               Asian    0  20564-1  \n",
              "3         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "4         Not Hispanic or Latino               Asian    0  11556-8  \n",
              "5         Not Hispanic or Latino               Asian    0   1959-6  \n",
              "...                          ...                 ...  ...      ...  \n",
              "14244781      Hispanic or Latino  White or Caucasian   53   2160-0  \n",
              "14244782                 Unknown             Unknown   71   6690-2  \n",
              "14244783      Hispanic or Latino  White or Caucasian    2  43743-4  \n",
              "14244784  Not Hispanic or Latino  White or Caucasian   36    731-0  \n",
              "14244785      Hispanic or Latino             Unknown   13  25132-2  \n",
              "\n",
              "[13872056 rows x 10 columns]"
            ]
          },
          "execution_count": 90,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "clear_ranges"
      ],
      "id": "80d6ecd1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "87ce9540",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "def generateRanges(clear_ranges):\n",
        "  output_df = pd.DataFrame(columns = clear_ranges.columns)\n",
        "\n",
        "  for omopid in set(clear_ranges['OMOPid']):\n",
        "      curr_df = clear_ranges[clear_ranges['OMOPid'] == omopid]\n",
        "\n",
        "      # check if multiple units\n",
        "      if (len(set(curr_df['unit'])) < 2):\n",
        "          # merge similar ranges (+/- 10%)\n",
        "          merge_similar_ranges(curr_df, 0.25)\n",
        "          # change demographic to 'any' where applicable\n",
        "          cleaned = merge_demographics(curr_df)\n",
        "          # append to output\n",
        "          output_df = output_df.append(curr_df)\n",
        "          continue\n",
        "\n",
        "      # check if there is a dominant unit\n",
        "      if omopid not in unit_map:\n",
        "          # don't append if no dominant unit\n",
        "          continue\n",
        "\n",
        "      # convert if necessary\n",
        "      converted = convertUnits(curr_df)\n",
        "      # merge similar ranges (+/- 10%)\n",
        "      merge_similar_ranges(converted, 0.25)\n",
        "      # change demographic to 'any' where applicable\n",
        "      cleaned = merge_demographics(converted)\n",
        "      # append to output\n",
        "      output_df = output_df.append(cleaned)\n"
      ],
      "id": "87ce9540"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c231eb1d"
      },
      "outputs": [],
      "source": [
        "output_df = generateRanges(clear_ranges)"
      ],
      "id": "c231eb1d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "541b95ab"
      },
      "outputs": [],
      "source": [
        "rows = len(output_df)\n",
        "output_df.to_csv('cleaned_ehr_' + str(rows) + '_rows.csv')"
      ],
      "id": "541b95ab"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2f64cfe4"
      },
      "source": [
        "# Outputs for different parameters:\n",
        "5,840,912 rows for\n",
        "- merge_similar_ranges(..., ci = 0.10)\n",
        "- all_ages(..., bin_size = 10)\n",
        "- getMaxUnit(..., 0.75)\n",
        "\n",
        "5,840,364 rows for\n",
        "- merge_similar_ranges(..., ci = 0.25)\n",
        "- all_ages(..., bin_size = 30)\n",
        "- all_ages: threshold = 0.50\n",
        "- getMaxUnit(..., 0.55)"
      ],
      "id": "2f64cfe4"
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "aa8a5df2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "161802b9-5c6e-4433-a660-ea105b247269"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-2-999ca6afe5c2>:2: DtypeWarning: Columns (9,11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  cleaned_ehr = pd.read_csv(filepath + 'cleaned_ehr.csv')\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           OMOPid manufacturer    unit  range_low  range_high  gender  \\\n",
              "Index                                                                   \n",
              "1107112   3006467      LAB3381  umol/l      31.26       84.11  Female   \n",
              "5202898   3006467       LC4764  umol/l      35.60      101.60  Female   \n",
              "6504574   3006467      LAB3381  umol/l      32.10       71.90  Female   \n",
              "8245085   3006467      LAB3381  umol/l      31.26       84.11    Male   \n",
              "13622773  3006467      LAB3381  umol/l      31.26       84.11    Male   \n",
              "...           ...          ...     ...        ...         ...     ...   \n",
              "14239825  3016699      LAB3020   mg/dl      70.00      139.00  Female   \n",
              "14242860  3016699      LAB3020   mg/dl      70.00      139.00  Female   \n",
              "14243816  3016699      LAB3020   mg/dl      70.00      139.00  Female   \n",
              "14243911  3016699      LAB3020   mg/dl      70.00      139.00  Female   \n",
              "14244323  3016699      LAB3020   mg/dl      70.00      139.00  Female   \n",
              "\n",
              "                       ethnicity                                       race  \\\n",
              "Index                                                                         \n",
              "1107112   Not Hispanic or Latino                         White or Caucasian   \n",
              "5202898   Not Hispanic or Latino                  Black or African American   \n",
              "6504574   Not Hispanic or Latino                         White or Caucasian   \n",
              "8245085   Not Hispanic or Latino                         White or Caucasian   \n",
              "13622773  Not Hispanic or Latino                                      Asian   \n",
              "...                          ...                                        ...   \n",
              "14239825  Not Hispanic or Latino                                    Unknown   \n",
              "14242860  Not Hispanic or Latino                         White or Caucasian   \n",
              "14243816  Not Hispanic or Latino                                      Asian   \n",
              "14243911  Not Hispanic or Latino  Native Hawaiian or Other Pacific Islander   \n",
              "14244323  Not Hispanic or Latino                         White or Caucasian   \n",
              "\n",
              "         age  LOINCid age_range  \n",
              "Index                            \n",
              "1107112   19  30195-2       NaN  \n",
              "5202898   38  30195-2       NaN  \n",
              "6504574   14  30195-2       NaN  \n",
              "8245085   58  30195-2       NaN  \n",
              "13622773  68  30195-2       NaN  \n",
              "...       ..      ...       ...  \n",
              "14239825  41   1504-0       NaN  \n",
              "14242860  20   1504-0       NaN  \n",
              "14243816  33   1504-0       NaN  \n",
              "14243911  44   1504-0       NaN  \n",
              "14244323  28   1504-0       NaN  \n",
              "\n",
              "[5840912 rows x 11 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-62fea1ed-a9d0-4b0c-af9f-a1a6eb413868\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OMOPid</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>unit</th>\n",
              "      <th>range_low</th>\n",
              "      <th>range_high</th>\n",
              "      <th>gender</th>\n",
              "      <th>ethnicity</th>\n",
              "      <th>race</th>\n",
              "      <th>age</th>\n",
              "      <th>LOINCid</th>\n",
              "      <th>age_range</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Index</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1107112</th>\n",
              "      <td>3006467</td>\n",
              "      <td>LAB3381</td>\n",
              "      <td>umol/l</td>\n",
              "      <td>31.26</td>\n",
              "      <td>84.11</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>19</td>\n",
              "      <td>30195-2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5202898</th>\n",
              "      <td>3006467</td>\n",
              "      <td>LC4764</td>\n",
              "      <td>umol/l</td>\n",
              "      <td>35.60</td>\n",
              "      <td>101.60</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Black or African American</td>\n",
              "      <td>38</td>\n",
              "      <td>30195-2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6504574</th>\n",
              "      <td>3006467</td>\n",
              "      <td>LAB3381</td>\n",
              "      <td>umol/l</td>\n",
              "      <td>32.10</td>\n",
              "      <td>71.90</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>14</td>\n",
              "      <td>30195-2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8245085</th>\n",
              "      <td>3006467</td>\n",
              "      <td>LAB3381</td>\n",
              "      <td>umol/l</td>\n",
              "      <td>31.26</td>\n",
              "      <td>84.11</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>58</td>\n",
              "      <td>30195-2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13622773</th>\n",
              "      <td>3006467</td>\n",
              "      <td>LAB3381</td>\n",
              "      <td>umol/l</td>\n",
              "      <td>31.26</td>\n",
              "      <td>84.11</td>\n",
              "      <td>Male</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>68</td>\n",
              "      <td>30195-2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14239825</th>\n",
              "      <td>3016699</td>\n",
              "      <td>LAB3020</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>70.00</td>\n",
              "      <td>139.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Unknown</td>\n",
              "      <td>41</td>\n",
              "      <td>1504-0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14242860</th>\n",
              "      <td>3016699</td>\n",
              "      <td>LAB3020</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>70.00</td>\n",
              "      <td>139.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>20</td>\n",
              "      <td>1504-0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14243816</th>\n",
              "      <td>3016699</td>\n",
              "      <td>LAB3020</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>70.00</td>\n",
              "      <td>139.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Asian</td>\n",
              "      <td>33</td>\n",
              "      <td>1504-0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14243911</th>\n",
              "      <td>3016699</td>\n",
              "      <td>LAB3020</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>70.00</td>\n",
              "      <td>139.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>Native Hawaiian or Other Pacific Islander</td>\n",
              "      <td>44</td>\n",
              "      <td>1504-0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14244323</th>\n",
              "      <td>3016699</td>\n",
              "      <td>LAB3020</td>\n",
              "      <td>mg/dl</td>\n",
              "      <td>70.00</td>\n",
              "      <td>139.00</td>\n",
              "      <td>Female</td>\n",
              "      <td>Not Hispanic or Latino</td>\n",
              "      <td>White or Caucasian</td>\n",
              "      <td>28</td>\n",
              "      <td>1504-0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5840912 rows × 11 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-62fea1ed-a9d0-4b0c-af9f-a1a6eb413868')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-62fea1ed-a9d0-4b0c-af9f-a1a6eb413868 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-62fea1ed-a9d0-4b0c-af9f-a1a6eb413868');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "import pandas as pd\n",
        "cleaned_ehr = pd.read_csv(filepath + 'cleaned_ehr.csv')\n",
        "cleaned_ehr.set_index(\"Unnamed: 0\", inplace=True)\n",
        "cleaned_ehr.index.name = \"Index\"\n",
        "cleaned_ehr"
      ],
      "id": "aa8a5df2"
    },
    {
      "cell_type": "code",
      "source": [
        "len(set(cleaned_ehr['LOINCid']))"
      ],
      "metadata": {
        "id": "6jw3QlczyvTo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae69551c-48fd-4442-9b71-9e420c8f50fd"
      },
      "id": "6jw3QlczyvTo",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "868"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ce1kUy6p7Swo"
      },
      "id": "ce1kUy6p7Swo",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}