mimic_Chapter_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "print(csv)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CJ0xyU0smVxL",
        "outputId": "fa8705c1-584a-487d-86e8-4e345d6c8de8"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       caregiver_id\n",
            "0                10\n",
            "1                17\n",
            "2                20\n",
            "3                25\n",
            "4                27\n",
            "...             ...\n",
            "15463         99973\n",
            "15464         99978\n",
            "15465         99986\n",
            "15466         99992\n",
            "15467         99999\n",
            "\n",
            "[15468 rows x 1 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "16HXiBovmBqU"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "csv = pd.read_parquet('caregiver.parquet', engine='pyarrow')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_parquet('caregiver.parquet', engine='pyarrow')\n"
      ],
      "metadata": {
        "id": "NuC_bxMnm-05"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ah_33-QoncHz",
        "outputId": "b7ae96f9-7bd2-4170-afcf-5b929249bd73"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   caregiver_id\n",
            "0            10\n",
            "1            17\n",
            "2            20\n",
            "3            25\n",
            "4            27\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 15468 entries, 0 to 15467\n",
            "Data columns (total 1 columns):\n",
            " #   Column        Non-Null Count  Dtype\n",
            "---  ------        --------------  -----\n",
            " 0   caregiver_id  15468 non-null  int32\n",
            "dtypes: int32(1)\n",
            "memory usage: 60.5 KB\n",
            "None\n",
            "       caregiver_id\n",
            "count  15468.000000\n",
            "mean   49911.511572\n",
            "std    28817.327784\n",
            "min       10.000000\n",
            "25%    24948.750000\n",
            "50%    49792.000000\n",
            "75%    74965.500000\n",
            "max    99999.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "df.dropna(inplace=True)\n",
        "\n",
        "\n",
        "df.fillna(df.mean(), inplace=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x1WRCADGnfnn",
        "outputId": "f698e509-d464-4d93-a497-092ce58927aa"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "caregiver_id    0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('d_hcpcs.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "d5HTpvvLn74z"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZCc223rIo1ET",
        "outputId": "4b06f6e0-af86-4f35-f7b4-93332978c0b7"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    code  category long_description   short_description\n",
            "0  00000       NaN             None        Invalid Code\n",
            "1  0001F       2.0             None  Composite measures\n",
            "2  0002F       2.0             None  Composite measures\n",
            "3  0003F       2.0             None  Composite measures\n",
            "4  0004F       2.0             None  Composite measures\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 89200 entries, 0 to 89199\n",
            "Data columns (total 4 columns):\n",
            " #   Column             Non-Null Count  Dtype  \n",
            "---  ------             --------------  -----  \n",
            " 0   code               89200 non-null  object \n",
            " 1   category           82356 non-null  float64\n",
            " 2   long_description   6800 non-null   object \n",
            " 3   short_description  89200 non-null  object \n",
            "dtypes: float64(1), object(3)\n",
            "memory usage: 2.7+ MB\n",
            "None\n",
            "           category\n",
            "count  82356.000000\n",
            "mean       1.029348\n",
            "std        0.202350\n",
            "min        1.000000\n",
            "25%        1.000000\n",
            "50%        1.000000\n",
            "75%        1.000000\n",
            "max        3.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)\n",
        "\n",
        "\n",
        "df.fillna(df.mean(), inplace=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "78SiMFcqo6rK",
        "outputId": "b9eff141-90fa-4b62-de6b-210196e4be42"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "code                     0\n",
            "category              6844\n",
            "long_description     82400\n",
            "short_description        0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_parquet('d_hcpcs.parquet', engine='pyarrow')\n",
        "\n",
        "\n",
        "print(\"Columns in the dataset:\")\n",
        "print(df.columns)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_G4fbQoipTV-",
        "outputId": "3ee80d76-31b3-4316-ccdf-f1d11ddd8fcd"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Columns in the dataset:\n",
            "Index(['code', 'category', 'long_description', 'short_description'], dtype='object')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('d_icd_procedures.parquet', engine='pyarrow')\n"
      ],
      "metadata": {
        "id": "M7CWwAG0sS8S"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(\"First few rows:\")\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(\"\\nInformation about the dataset:\")\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(\"\\nSummary statistics:\")\n",
        "print(df.describe())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8vFQcmypszfc",
        "outputId": "0bbdd86c-abec-46f0-837e-b4921f59beec"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First few rows:\n",
            "Empty DataFrame\n",
            "Columns: [code, category, long_description, short_description]\n",
            "Index: []\n",
            "\n",
            "Information about the dataset:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Index: 0 entries\n",
            "Data columns (total 4 columns):\n",
            " #   Column             Non-Null Count  Dtype  \n",
            "---  ------             --------------  -----  \n",
            " 0   code               0 non-null      object \n",
            " 1   category           0 non-null      float64\n",
            " 2   long_description   0 non-null      object \n",
            " 3   short_description  0 non-null      object \n",
            "dtypes: float64(1), object(3)\n",
            "memory usage: 0.0+ bytes\n",
            "None\n",
            "\n",
            "Summary statistics:\n",
            "       category\n",
            "count       0.0\n",
            "mean        NaN\n",
            "std         NaN\n",
            "min         NaN\n",
            "25%         NaN\n",
            "50%         NaN\n",
            "75%         NaN\n",
            "max         NaN\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(\"\\nMissing values:\")\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FXZxKfJAtohv",
        "outputId": "4cd67688-3c34-46f5-b0c1-69116a7eba40"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Missing values:\n",
            "itemid                0\n",
            "label                 0\n",
            "abbreviation          0\n",
            "linksto               0\n",
            "category              0\n",
            "unitname           2422\n",
            "param_type            0\n",
            "lownormalvalue     3995\n",
            "highnormalvalue    3992\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_parquet('d_items.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "F5yLIGAsuFDk"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(\"First few rows:\")\n",
        "print(df.head())\n",
        "\n",
        "print(\"\\nInformation about the dataset:\")\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(\"\\nSummary statistics:\")\n",
        "print(df.describe())\n",
        "\n",
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kLUTbT2duiIi",
        "outputId": "75ef5a8a-0559-4d98-e949-89efa826f3da"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First few rows:\n",
            "   itemid                    label        abbreviation         linksto  \\\n",
            "0  220001             Problem List        Problem List     chartevents   \n",
            "1  220003       ICU Admission date  ICU Admission date  datetimeevents   \n",
            "2  220045               Heart Rate                  HR     chartevents   \n",
            "3  220046  Heart rate Alarm - High     HR Alarm - High     chartevents   \n",
            "4  220047   Heart Rate Alarm - Low      HR Alarm - Low     chartevents   \n",
            "\n",
            "              category unitname     param_type  lownormalvalue  \\\n",
            "0              General     None           Text             NaN   \n",
            "1                  ADT     None  Date and time             NaN   \n",
            "2  Routine Vital Signs      bpm        Numeric             NaN   \n",
            "3               Alarms      bpm        Numeric             NaN   \n",
            "4               Alarms      bpm        Numeric             NaN   \n",
            "\n",
            "   highnormalvalue  \n",
            "0              NaN  \n",
            "1              NaN  \n",
            "2              NaN  \n",
            "3              NaN  \n",
            "4              NaN  \n",
            "\n",
            "Information about the dataset:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 4014 entries, 0 to 4013\n",
            "Data columns (total 9 columns):\n",
            " #   Column           Non-Null Count  Dtype  \n",
            "---  ------           --------------  -----  \n",
            " 0   itemid           4014 non-null   int32  \n",
            " 1   label            4014 non-null   object \n",
            " 2   abbreviation     4014 non-null   object \n",
            " 3   linksto          4014 non-null   object \n",
            " 4   category         4014 non-null   object \n",
            " 5   unitname         1592 non-null   object \n",
            " 6   param_type       4014 non-null   object \n",
            " 7   lownormalvalue   19 non-null     float32\n",
            " 8   highnormalvalue  22 non-null     float32\n",
            "dtypes: float32(2), int32(1), object(6)\n",
            "memory usage: 235.3+ KB\n",
            "None\n",
            "\n",
            "Summary statistics:\n",
            "              itemid  lownormalvalue  highnormalvalue\n",
            "count    4014.000000       19.000000        22.000000\n",
            "mean   226536.567763       65.000000       133.640915\n",
            "std      2467.222449      107.356621       253.076004\n",
            "min    220001.000000       -2.000000         8.000000\n",
            "25%    224973.250000        4.000000        15.000000\n",
            "50%    226862.500000       10.000000        22.500000\n",
            "75%    228547.750000       60.000000        87.500000\n",
            "max    230085.000000      299.000000       750.000000\n",
            "itemid                0\n",
            "label                 0\n",
            "abbreviation          0\n",
            "linksto               0\n",
            "category              0\n",
            "unitname           2422\n",
            "param_type            0\n",
            "lownormalvalue     3995\n",
            "highnormalvalue    3992\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('d_labitems.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "0-xUGKCZvM2a"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(\"First few rows:\")\n",
        "print(df.head())\n",
        "\n",
        "print(\"\\nInformation about the dataset:\")\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(\"\\nSummary statistics:\")\n",
        "print(df.describe())\n",
        "\n",
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94t2-eT5vhul",
        "outputId": "74d9c8e6-38b1-4d4c-b510-2d10576887c1"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First few rows:\n",
            "   itemid                                label  fluid   category\n",
            "0   50801           Alveolar-arterial Gradient  Blood  Blood Gas\n",
            "1   50802                          Base Excess  Blood  Blood Gas\n",
            "2   50803  Calculated Bicarbonate, Whole Blood  Blood  Blood Gas\n",
            "3   50804                 Calculated Total CO2  Blood  Blood Gas\n",
            "4   50805                    Carboxyhemoglobin  Blood  Blood Gas\n",
            "\n",
            "Information about the dataset:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1622 entries, 0 to 1621\n",
            "Data columns (total 4 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   itemid    1622 non-null   int32 \n",
            " 1   label     1619 non-null   object\n",
            " 2   fluid     1622 non-null   object\n",
            " 3   category  1622 non-null   object\n",
            "dtypes: int32(1), object(3)\n",
            "memory usage: 44.5+ KB\n",
            "None\n",
            "\n",
            "Summary statistics:\n",
            "             itemid\n",
            "count   1622.000000\n",
            "mean   51706.821208\n",
            "std      573.623215\n",
            "min    50801.000000\n",
            "25%    51220.250000\n",
            "50%    51688.500000\n",
            "75%    52123.750000\n",
            "max    53154.000000\n",
            "itemid      0\n",
            "label       3\n",
            "fluid       0\n",
            "category    0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('hcpcsevents.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "agSw5xZ2vlNn"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "print(df.describe())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WmcwpraUwXUz",
        "outputId": "b8476777-1173-4673-9cbb-358d7d8419a8"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   subject_id   hadm_id   chartdate hcpcs_cd  seq_num  \\\n",
            "0    10000068  25022803  2160-03-04    99218        1   \n",
            "1    10000084  29888819  2160-12-28    G0378        1   \n",
            "2    10000108  27250926  2163-09-27    99219        1   \n",
            "3    10000117  22927623  2181-11-15    43239        1   \n",
            "4    10000117  22927623  2181-11-15    G0378        2   \n",
            "\n",
            "               short_description  \n",
            "0  Hospital observation services  \n",
            "1    Hospital observation per hr  \n",
            "2  Hospital observation services  \n",
            "3               Digestive system  \n",
            "4    Hospital observation per hr  \n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 150771 entries, 0 to 150770\n",
            "Data columns (total 6 columns):\n",
            " #   Column             Non-Null Count   Dtype \n",
            "---  ------             --------------   ----- \n",
            " 0   subject_id         150771 non-null  int32 \n",
            " 1   hadm_id            150771 non-null  int32 \n",
            " 2   chartdate          150771 non-null  object\n",
            " 3   hcpcs_cd           150771 non-null  object\n",
            " 4   seq_num            150771 non-null  int32 \n",
            " 5   short_description  150771 non-null  object\n",
            "dtypes: int32(3), object(3)\n",
            "memory usage: 5.2+ MB\n",
            "None\n",
            "         subject_id       hadm_id        seq_num\n",
            "count  1.507710e+05  1.507710e+05  150771.000000\n",
            "mean   1.500822e+07  2.501134e+07       1.356899\n",
            "std    2.870753e+06  2.884234e+06       0.871453\n",
            "min    1.000007e+07  2.000003e+07       1.000000\n",
            "25%    1.252834e+07  2.250509e+07       1.000000\n",
            "50%    1.501616e+07  2.503036e+07       1.000000\n",
            "75%    1.748011e+07  2.750074e+07       1.000000\n",
            "max    1.999978e+07  2.999993e+07      15.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dKeeTYkLwc0b",
        "outputId": "d1389326-bfbb-4b3c-e881-347015e3c870"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "subject_id           0\n",
            "hadm_id              0\n",
            "chartdate            0\n",
            "hcpcs_cd             0\n",
            "seq_num              0\n",
            "short_description    0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_parquet('poe_detail.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "8s7HOOqCyeM1"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AAWHKA6c0bW5",
        "outputId": "62ed30c2-393b-4653-e329-e5018f766eee"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "         poe_id  poe_seq  subject_id          field_name         field_value\n",
            "0  10000032-104      104    10000032      Admit category  Admit to inpatient\n",
            "1  10000032-104      104    10000032            Admit to            Medicine\n",
            "2  10000032-109      109    10000032      Admit category  Admit to inpatient\n",
            "3  10000032-109      109    10000032  Discharge Planning           Finalized\n",
            "4  10000032-111      111    10000032      Admit category  Admit to inpatient\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 3879418 entries, 0 to 3879417\n",
            "Data columns (total 5 columns):\n",
            " #   Column       Dtype \n",
            "---  ------       ----- \n",
            " 0   poe_id       object\n",
            " 1   poe_seq      int32 \n",
            " 2   subject_id   int32 \n",
            " 3   field_name   object\n",
            " 4   field_value  object\n",
            "dtypes: int32(2), object(3)\n",
            "memory usage: 118.4+ MB\n",
            "None\n",
            "            poe_seq    subject_id\n",
            "count  3.879418e+06  3.879418e+06\n",
            "mean   5.723801e+02  1.500660e+07\n",
            "std    9.699043e+02  2.880653e+06\n",
            "min    2.000000e+00  1.000003e+07\n",
            "25%    8.800000e+01  1.250128e+07\n",
            "50%    2.400000e+02  1.500530e+07\n",
            "75%    6.360000e+02  1.751005e+07\n",
            "max    2.008100e+04  1.999999e+07\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nyhH6wio0hd3",
        "outputId": "80f4c1c8-b072-4ce4-f36f-b9e54c38be84"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "poe_id         0\n",
            "poe_seq        0\n",
            "subject_id     0\n",
            "field_name     0\n",
            "field_value    0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('patients.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "9BVK78Oe0mY1"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FL4b-60N0ym6",
        "outputId": "eee3b402-1799-4b86-9db7-57349b22e145"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   subject_id gender  anchor_age  anchor_year anchor_year_group         dod\n",
            "0    10000032      F          52         2180       2014 - 2016  2180-09-09\n",
            "1    10000048      F          23         2126       2008 - 2010        None\n",
            "2    10000068      F          19         2160       2008 - 2010        None\n",
            "3    10000084      M          72         2160       2017 - 2019  2161-02-13\n",
            "4    10000102      F          27         2136       2008 - 2010        None\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 299712 entries, 0 to 299711\n",
            "Data columns (total 6 columns):\n",
            " #   Column             Non-Null Count   Dtype \n",
            "---  ------             --------------   ----- \n",
            " 0   subject_id         299712 non-null  int32 \n",
            " 1   gender             299712 non-null  object\n",
            " 2   anchor_age         299712 non-null  int16 \n",
            " 3   anchor_year        299712 non-null  int16 \n",
            " 4   anchor_year_group  299712 non-null  object\n",
            " 5   dod                29076 non-null   object\n",
            "dtypes: int16(2), int32(1), object(3)\n",
            "memory usage: 9.1+ MB\n",
            "None\n",
            "         subject_id     anchor_age    anchor_year\n",
            "count  2.997120e+05  299712.000000  299712.000000\n",
            "mean   1.500885e+07      48.539888    2151.124056\n",
            "std    2.885019e+06      20.895650      23.464598\n",
            "min    1.000003e+07      18.000000    2110.000000\n",
            "25%    1.250796e+07      29.000000    2131.000000\n",
            "50%    1.501416e+07      48.000000    2151.000000\n",
            "75%    1.750489e+07      65.000000    2171.000000\n",
            "max    1.999999e+07      91.000000    2208.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hS0cu2EG01bk",
        "outputId": "40d1c7ff-d8dd-452d-c4a7-3670f66b7da5"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "subject_id                0\n",
            "gender                    0\n",
            "anchor_age                0\n",
            "anchor_year               0\n",
            "anchor_year_group         0\n",
            "dod                  270636\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('outputevents.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "LJ9MVvvy09vK"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HJDQThQB1KAd",
        "outputId": "33797d18-930e-45c5-891b-dbc4802eed87"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   subject_id   hadm_id   stay_id  caregiver_id           charttime  \\\n",
            "0    10000032  29079034  39553978         88981 2180-07-23 15:00:00   \n",
            "1    10000980  26913865  39765666         36518 2189-06-27 09:08:00   \n",
            "2    10000980  26913865  39765666         36518 2189-06-27 09:08:00   \n",
            "3    10000980  26913865  39765666         36518 2189-06-27 11:00:00   \n",
            "4    10000980  26913865  39765666         36518 2189-06-27 13:00:00   \n",
            "\n",
            "            storetime  itemid  value valueuom  \n",
            "0 2180-07-23 16:00:00  226560  175.0       ml  \n",
            "1 2189-06-27 09:08:00  226559  450.0       ml  \n",
            "2 2189-06-27 09:08:00  226633  400.0       ml  \n",
            "3 2189-06-27 10:51:00  226559  600.0       ml  \n",
            "4 2189-06-27 12:55:00  226559  800.0       ml  \n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 4234967 entries, 0 to 4234966\n",
            "Data columns (total 9 columns):\n",
            " #   Column        Dtype         \n",
            "---  ------        -----         \n",
            " 0   subject_id    int32         \n",
            " 1   hadm_id       int32         \n",
            " 2   stay_id       int32         \n",
            " 3   caregiver_id  int32         \n",
            " 4   charttime     datetime64[us]\n",
            " 5   storetime     datetime64[us]\n",
            " 6   itemid        int32         \n",
            " 7   value         float32       \n",
            " 8   valueuom      object        \n",
            "dtypes: datetime64[us](2), float32(1), int32(5), object(1)\n",
            "memory usage: 193.9+ MB\n",
            "None\n",
            "         subject_id       hadm_id       stay_id  caregiver_id  \\\n",
            "count  4.234967e+06  4.234967e+06  4.234967e+06  4.234967e+06   \n",
            "mean   1.501570e+07  2.501139e+07  3.499809e+07  5.095068e+04   \n",
            "min    1.000003e+07  2.000009e+07  3.000015e+07  4.500000e+01   \n",
            "25%    1.251816e+07  2.252594e+07  3.247586e+07  2.639500e+04   \n",
            "50%    1.503547e+07  2.503694e+07  3.499598e+07  5.316400e+04   \n",
            "75%    1.750312e+07  2.749249e+07  3.749746e+07  7.791200e+04   \n",
            "max    1.999999e+07  2.999983e+07  3.999981e+07  9.994400e+04   \n",
            "std    2.883852e+06  2.875828e+06  2.895064e+06  2.894277e+04   \n",
            "\n",
            "                        charttime                   storetime        itemid  \\\n",
            "count                     4234967                     4234967  4.234967e+06   \n",
            "mean   2153-07-16 16:13:53.534115  2153-07-16 16:50:56.596751  2.265945e+05   \n",
            "min           2110-01-11 12:30:00         2110-01-11 12:50:00  2.265570e+05   \n",
            "25%           2133-05-22 17:00:00         2133-05-22 18:11:00  2.265590e+05   \n",
            "50%           2153-06-03 00:00:00         2153-06-03 00:15:00  2.265590e+05   \n",
            "75%           2173-09-11 09:00:00         2173-09-11 09:25:00  2.265600e+05   \n",
            "max           2211-11-11 15:32:00         2211-11-11 15:32:00  2.294140e+05   \n",
            "std                           NaN                         NaN  1.721959e+02   \n",
            "\n",
            "              value  \n",
            "count  4.234967e+06  \n",
            "mean   1.350025e+02  \n",
            "min   -3.765000e+03  \n",
            "25%    3.500000e+01  \n",
            "50%    7.500000e+01  \n",
            "75%    1.600000e+02  \n",
            "max    1.293850e+05  \n",
            "std    2.480186e+02  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sHRLsfHH1N1-",
        "outputId": "b4b93425-c9d6-4a0f-df9a-c730f9725e7e"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "subject_id      0\n",
            "hadm_id         0\n",
            "stay_id         0\n",
            "caregiver_id    0\n",
            "charttime       0\n",
            "storetime       0\n",
            "itemid          0\n",
            "value           0\n",
            "valueuom        0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "df = pd.read_parquet('omr.parquet', engine='pyarrow')"
      ],
      "metadata": {
        "id": "y0RbUJRP1Utf"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.head())\n",
        "\n",
        "\n",
        "print(df.info())\n",
        "\n",
        "\n",
        "print(df.describe())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bKh0zmDN1hyv",
        "outputId": "fe143e91-5828-43ee-cef1-3c1a24eb3d84"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   subject_id   chartdate  seq_num      result_name result_value\n",
            "0    10000032  2180-04-27        1   Blood Pressure       110/65\n",
            "1    10000032  2180-04-27        1     Weight (Lbs)           94\n",
            "2    10000032  2180-05-07        1      BMI (kg/m2)         18.0\n",
            "3    10000032  2180-05-07        1  Height (Inches)           60\n",
            "4    10000032  2180-05-07        1     Weight (Lbs)        92.15\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 6439169 entries, 0 to 6439168\n",
            "Data columns (total 5 columns):\n",
            " #   Column        Dtype \n",
            "---  ------        ----- \n",
            " 0   subject_id    int32 \n",
            " 1   chartdate     object\n",
            " 2   seq_num       int32 \n",
            " 3   result_name   object\n",
            " 4   result_value  object\n",
            "dtypes: int32(2), object(3)\n",
            "memory usage: 196.5+ MB\n",
            "None\n",
            "         subject_id       seq_num\n",
            "count  6.439169e+06  6.439169e+06\n",
            "mean   1.501668e+07  1.119485e+00\n",
            "std    2.881851e+06  1.044536e+00\n",
            "min    1.000003e+07  1.000000e+00\n",
            "25%    1.253175e+07  1.000000e+00\n",
            "50%    1.502411e+07  1.000000e+00\n",
            "75%    1.750936e+07  1.000000e+00\n",
            "max    1.999983e+07  6.700000e+01\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(df.isnull().sum())\n",
        "\n",
        "\n",
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K4ITsJ111l8n",
        "outputId": "e2e8c27e-a9da-48f8-a302-0f900df95d3e"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "subject_id      0\n",
            "chartdate       0\n",
            "seq_num         0\n",
            "result_name     0\n",
            "result_value    0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kbieJDGr3x8T"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}