Demographic_Clusters_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Purpose: Create clusters of patients based on all of the complete data in the dataset. \n",
    "\n",
    "**These clusters will be used when inputing missing data.** \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.metrics import silhouette_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load and Inspect Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>encounter_id</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>hospital_id</th>\n",
       "      <th>hospital_death</th>\n",
       "      <th>age</th>\n",
       "      <th>bmi</th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>ethnicity</th>\n",
       "      <th>gender</th>\n",
       "      <th>height</th>\n",
       "      <th>...</th>\n",
       "      <th>diabetes_mellitus</th>\n",
       "      <th>hepatic_failure</th>\n",
       "      <th>immunosuppression</th>\n",
       "      <th>leukemia</th>\n",
       "      <th>lymphoma</th>\n",
       "      <th>solid_tumor_with_metastasis</th>\n",
       "      <th>apache_3j_bodysystem</th>\n",
       "      <th>apache_2_bodysystem</th>\n",
       "      <th>patients_per_hospital</th>\n",
       "      <th>hospital_traffic_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>66154</td>\n",
       "      <td>25312</td>\n",
       "      <td>118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>22.73</td>\n",
       "      <td>0</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>M</td>\n",
       "      <td>180.3</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sepsis</td>\n",
       "      <td>Cardiovascular</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114252</td>\n",
       "      <td>59342</td>\n",
       "      <td>81</td>\n",
       "      <td>0.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>27.42</td>\n",
       "      <td>0</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>F</td>\n",
       "      <td>160.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Respiratory</td>\n",
       "      <td>Respiratory</td>\n",
       "      <td>580</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119783</td>\n",
       "      <td>50777</td>\n",
       "      <td>118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>31.95</td>\n",
       "      <td>0</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>F</td>\n",
       "      <td>172.7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Metabolic</td>\n",
       "      <td>Metabolic</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>79267</td>\n",
       "      <td>46918</td>\n",
       "      <td>118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>22.64</td>\n",
       "      <td>1</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>F</td>\n",
       "      <td>165.1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Cardiovascular</td>\n",
       "      <td>Cardiovascular</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>92056</td>\n",
       "      <td>34377</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>M</td>\n",
       "      <td>188.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Trauma</td>\n",
       "      <td>Trauma</td>\n",
       "      <td>323</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 188 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   encounter_id  patient_id  hospital_id  hospital_death   age    bmi  \\\n",
       "0         66154       25312          118             0.0  68.0  22.73   \n",
       "1        114252       59342           81             0.0  77.0  27.42   \n",
       "2        119783       50777          118             0.0  25.0  31.95   \n",
       "3         79267       46918          118             0.0  81.0  22.64   \n",
       "4         92056       34377           33             0.0  19.0    NaN   \n",
       "\n",
       "   elective_surgery  ethnicity gender  height  ... diabetes_mellitus  \\\n",
       "0                 0  Caucasian      M   180.3  ...               1.0   \n",
       "1                 0  Caucasian      F   160.0  ...               1.0   \n",
       "2                 0  Caucasian      F   172.7  ...               0.0   \n",
       "3                 1  Caucasian      F   165.1  ...               0.0   \n",
       "4                 0  Caucasian      M   188.0  ...               0.0   \n",
       "\n",
       "  hepatic_failure  immunosuppression leukemia lymphoma  \\\n",
       "0             0.0                0.0      0.0      0.0   \n",
       "1             0.0                0.0      0.0      0.0   \n",
       "2             0.0                0.0      0.0      0.0   \n",
       "3             0.0                0.0      0.0      0.0   \n",
       "4             0.0                0.0      0.0      0.0   \n",
       "\n",
       "   solid_tumor_with_metastasis  apache_3j_bodysystem  apache_2_bodysystem  \\\n",
       "0                          0.0                Sepsis       Cardiovascular   \n",
       "1                          0.0           Respiratory          Respiratory   \n",
       "2                          0.0             Metabolic            Metabolic   \n",
       "3                          0.0        Cardiovascular       Cardiovascular   \n",
       "4                          0.0                Trauma               Trauma   \n",
       "\n",
       "   patients_per_hospital  hospital_traffic_bin  \n",
       "0                   4333                     5  \n",
       "1                    580                     1  \n",
       "2                   4333                     5  \n",
       "3                   4333                     5  \n",
       "4                    323                     1  \n",
       "\n",
       "[5 rows x 188 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load data -  merged training and test data - selected columns with bins\n",
    "# Added column for hospital bins (binned on # of patients/hospital) 'hospital_traffic_bin'\n",
    "with open('df_hospital_bins2.pickle', 'rb') as read_file:\n",
    "    df_bins = pickle.load(read_file)\n",
    "df_bins.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for missing data\n",
    "df_bins['patients_per_hospital'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bins['hospital_traffic_bin'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>encounter_id</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>hospital_id</th>\n",
       "      <th>hospital_death</th>\n",
       "      <th>age</th>\n",
       "      <th>bmi</th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>height</th>\n",
       "      <th>icu_id</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>...</th>\n",
       "      <th>apache_4a_icu_death_prob</th>\n",
       "      <th>aids</th>\n",
       "      <th>cirrhosis</th>\n",
       "      <th>diabetes_mellitus</th>\n",
       "      <th>hepatic_failure</th>\n",
       "      <th>immunosuppression</th>\n",
       "      <th>leukemia</th>\n",
       "      <th>lymphoma</th>\n",
       "      <th>solid_tumor_with_metastasis</th>\n",
       "      <th>patients_per_hospital</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>131021.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>91713.000000</td>\n",
       "      <td>125139.000000</td>\n",
       "      <td>126506.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>128937.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>117896.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>130127.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>65528.171194</td>\n",
       "      <td>65527.414643</td>\n",
       "      <td>106.099259</td>\n",
       "      <td>0.086302</td>\n",
       "      <td>62.009965</td>\n",
       "      <td>29.113833</td>\n",
       "      <td>0.188588</td>\n",
       "      <td>169.604665</td>\n",
       "      <td>662.060212</td>\n",
       "      <td>0.833905</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042732</td>\n",
       "      <td>0.001030</td>\n",
       "      <td>0.016084</td>\n",
       "      <td>0.216312</td>\n",
       "      <td>0.013602</td>\n",
       "      <td>0.025675</td>\n",
       "      <td>0.007308</td>\n",
       "      <td>0.004188</td>\n",
       "      <td>0.020857</td>\n",
       "      <td>1527.399249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>37832.833351</td>\n",
       "      <td>37831.357384</td>\n",
       "      <td>63.493396</td>\n",
       "      <td>0.280811</td>\n",
       "      <td>16.797485</td>\n",
       "      <td>8.266435</td>\n",
       "      <td>0.391183</td>\n",
       "      <td>10.834117</td>\n",
       "      <td>304.224462</td>\n",
       "      <td>2.503083</td>\n",
       "      <td>...</td>\n",
       "      <td>0.223015</td>\n",
       "      <td>0.032074</td>\n",
       "      <td>0.125800</td>\n",
       "      <td>0.411731</td>\n",
       "      <td>0.115833</td>\n",
       "      <td>0.158164</td>\n",
       "      <td>0.085176</td>\n",
       "      <td>0.064581</td>\n",
       "      <td>0.142905</td>\n",
       "      <td>1104.982130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>14.844926</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>137.200000</td>\n",
       "      <td>82.000000</td>\n",
       "      <td>-82.028472</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>32763.000000</td>\n",
       "      <td>32764.000000</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>23.598616</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>162.500000</td>\n",
       "      <td>427.000000</td>\n",
       "      <td>0.045139</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>673.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>65527.000000</td>\n",
       "      <td>65529.000000</td>\n",
       "      <td>112.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>27.573696</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>170.100000</td>\n",
       "      <td>653.000000</td>\n",
       "      <td>0.154861</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1260.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>98294.000000</td>\n",
       "      <td>98290.000000</td>\n",
       "      <td>165.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>32.812500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>177.800000</td>\n",
       "      <td>969.000000</td>\n",
       "      <td>0.422917</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2470.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>131051.000000</td>\n",
       "      <td>131051.000000</td>\n",
       "      <td>204.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>67.814990</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>195.590000</td>\n",
       "      <td>1111.000000</td>\n",
       "      <td>175.627778</td>\n",
       "      <td>...</td>\n",
       "      <td>0.970000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4333.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 179 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        encounter_id     patient_id    hospital_id  hospital_death  \\\n",
       "count  131021.000000  131021.000000  131021.000000    91713.000000   \n",
       "mean    65528.171194   65527.414643     106.099259        0.086302   \n",
       "std     37832.833351   37831.357384      63.493396        0.280811   \n",
       "min         1.000000       1.000000       1.000000        0.000000   \n",
       "25%     32763.000000   32764.000000      49.000000        0.000000   \n",
       "50%     65527.000000   65529.000000     112.000000        0.000000   \n",
       "75%     98294.000000   98290.000000     165.000000        0.000000   \n",
       "max    131051.000000  131051.000000     204.000000        1.000000   \n",
       "\n",
       "                 age            bmi  elective_surgery         height  \\\n",
       "count  125139.000000  126506.000000     131021.000000  128937.000000   \n",
       "mean       62.009965      29.113833          0.188588     169.604665   \n",
       "std        16.797485       8.266435          0.391183      10.834117   \n",
       "min        16.000000      14.844926          0.000000     137.200000   \n",
       "25%        52.000000      23.598616          0.000000     162.500000   \n",
       "50%        64.000000      27.573696          0.000000     170.100000   \n",
       "75%        75.000000      32.812500          0.000000     177.800000   \n",
       "max        89.000000      67.814990          1.000000     195.590000   \n",
       "\n",
       "              icu_id  pre_icu_los_days  ...  apache_4a_icu_death_prob  \\\n",
       "count  131021.000000     131021.000000  ...             117896.000000   \n",
       "mean      662.060212          0.833905  ...                  0.042732   \n",
       "std       304.224462          2.503083  ...                  0.223015   \n",
       "min        82.000000        -82.028472  ...                 -1.000000   \n",
       "25%       427.000000          0.045139  ...                  0.010000   \n",
       "50%       653.000000          0.154861  ...                  0.020000   \n",
       "75%       969.000000          0.422917  ...                  0.060000   \n",
       "max      1111.000000        175.627778  ...                  0.970000   \n",
       "\n",
       "                aids      cirrhosis  diabetes_mellitus  hepatic_failure  \\\n",
       "count  130127.000000  130127.000000      130127.000000    130127.000000   \n",
       "mean        0.001030       0.016084           0.216312         0.013602   \n",
       "std         0.032074       0.125800           0.411731         0.115833   \n",
       "min         0.000000       0.000000           0.000000         0.000000   \n",
       "25%         0.000000       0.000000           0.000000         0.000000   \n",
       "50%         0.000000       0.000000           0.000000         0.000000   \n",
       "75%         0.000000       0.000000           0.000000         0.000000   \n",
       "max         1.000000       1.000000           1.000000         1.000000   \n",
       "\n",
       "       immunosuppression       leukemia       lymphoma  \\\n",
       "count      130127.000000  130127.000000  130127.000000   \n",
       "mean            0.025675       0.007308       0.004188   \n",
       "std             0.158164       0.085176       0.064581   \n",
       "min             0.000000       0.000000       0.000000   \n",
       "25%             0.000000       0.000000       0.000000   \n",
       "50%             0.000000       0.000000       0.000000   \n",
       "75%             0.000000       0.000000       0.000000   \n",
       "max             1.000000       1.000000       1.000000   \n",
       "\n",
       "       solid_tumor_with_metastasis  patients_per_hospital  \n",
       "count                130127.000000          131021.000000  \n",
       "mean                      0.020857            1527.399249  \n",
       "std                       0.142905            1104.982130  \n",
       "min                       0.000000               2.000000  \n",
       "25%                       0.000000             673.000000  \n",
       "50%                       0.000000            1260.000000  \n",
       "75%                       0.000000            2470.000000  \n",
       "max                       1.000000            4333.000000  \n",
       "\n",
       "[8 rows x 179 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bins.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  2. Select Variables to use for Clustering\n",
    "\n",
    "It's important for these variables to: \n",
    "* Have complete information for the training set and test set \n",
    "* Missing <1% of the training dataset is probably okay, but we'll have to make predictions for all of the test set - so I think we need info for all test set in order to cluster them.  \n",
    "\n",
    "Some variables that seem good to include, but have questionable levels of missing data include: \n",
    "* gender - 0.03/0.04\n",
    "* d1_heartrate_max\n",
    "* d1_sysbp_max\n",
    "* icu_admit_source\n",
    "* ethnicity - 1.45 / 1.91\n",
    "* weight - 2.97 /1.94\n",
    "* bmi - 3.74/ 2.76 \n",
    "* age - 4.61 / 4.21 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>encounter_id</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>hospital_id</th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>icu_id</th>\n",
       "      <th>icu_stay_type</th>\n",
       "      <th>icu_type</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>readmission_status</th>\n",
       "      <th>apache_post_operative</th>\n",
       "      <th>patients_per_hospital</th>\n",
       "      <th>hospital_traffic_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>66154</td>\n",
       "      <td>25312</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>92</td>\n",
       "      <td>admit</td>\n",
       "      <td>CTICU</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114252</td>\n",
       "      <td>59342</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>90</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.927778</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>580</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>119783</td>\n",
       "      <td>50777</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>93</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>79267</td>\n",
       "      <td>46918</td>\n",
       "      <td>118</td>\n",
       "      <td>1</td>\n",
       "      <td>92</td>\n",
       "      <td>admit</td>\n",
       "      <td>CTICU</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>92056</td>\n",
       "      <td>34377</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.073611</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>323</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   encounter_id  patient_id  hospital_id  elective_surgery  icu_id  \\\n",
       "0         66154       25312          118                 0      92   \n",
       "1        114252       59342           81                 0      90   \n",
       "2        119783       50777          118                 0      93   \n",
       "3         79267       46918          118                 1      92   \n",
       "4         92056       34377           33                 0      91   \n",
       "\n",
       "  icu_stay_type      icu_type  pre_icu_los_days  readmission_status  \\\n",
       "0         admit         CTICU          0.541667                   0   \n",
       "1         admit  Med-Surg ICU          0.927778                   0   \n",
       "2         admit  Med-Surg ICU          0.000694                   0   \n",
       "3         admit         CTICU          0.000694                   0   \n",
       "4         admit  Med-Surg ICU          0.073611                   0   \n",
       "\n",
       "   apache_post_operative  patients_per_hospital hospital_traffic_bin  \n",
       "0                      0                   4333                    5  \n",
       "1                      0                    580                    1  \n",
       "2                      0                   4333                    5  \n",
       "3                      1                   4333                    5  \n",
       "4                      0                    323                    1  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the columns that contain missing information (any missing info)\n",
    "df_no_nulls = df_bins.dropna(axis=1)\n",
    "df_no_nulls.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 131021 entries, 0 to 131020\n",
      "Data columns (total 12 columns):\n",
      "encounter_id             131021 non-null int64\n",
      "patient_id               131021 non-null int64\n",
      "hospital_id              131021 non-null int64\n",
      "elective_surgery         131021 non-null int64\n",
      "icu_id                   131021 non-null int64\n",
      "icu_stay_type            131021 non-null object\n",
      "icu_type                 131021 non-null object\n",
      "pre_icu_los_days         131021 non-null float64\n",
      "readmission_status       131021 non-null int64\n",
      "apache_post_operative    131021 non-null int64\n",
      "patients_per_hospital    131021 non-null int64\n",
      "hospital_traffic_bin     131021 non-null category\n",
      "dtypes: category(1), float64(1), int64(8), object(2)\n",
      "memory usage: 12.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_no_nulls.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "encounter_id             131021\n",
       "patient_id               131021\n",
       "hospital_id                 204\n",
       "elective_surgery              2\n",
       "icu_id                      328\n",
       "icu_stay_type                 3\n",
       "icu_type                      8\n",
       "pre_icu_los_days          11380\n",
       "readmission_status            1\n",
       "apache_post_operative         2\n",
       "patients_per_hospital       187\n",
       "hospital_traffic_bin          5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_no_nulls.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2b. Remove additional columns we don't want to include for clustering:\n",
    "* hospital_death -  we dont' want the target to impact the clusters\n",
    "* patient and encounter ids - don't matter for clustering since they're all be unique\n",
    "* readmission_status - all values are the same \n",
    "* patients_per_hospital - use hospital_traffic bins instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clustering = df_bins[['elective_surgery', 'icu_stay_type', 'icu_type',\n",
    "                         'pre_icu_los_days', 'apache_post_operative', 'hospital_traffic_bin']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['elective_surgery', 'icu_stay_type', 'icu_type', 'pre_icu_los_days',\n",
       "       'apache_post_operative', 'hospital_traffic_bin'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>apache_post_operative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>131021.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "      <td>131021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.188588</td>\n",
       "      <td>0.833905</td>\n",
       "      <td>0.205746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.391183</td>\n",
       "      <td>2.503083</td>\n",
       "      <td>0.404247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-82.028472</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045139</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.154861</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.422917</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>175.627778</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       elective_surgery  pre_icu_los_days  apache_post_operative\n",
       "count     131021.000000     131021.000000          131021.000000\n",
       "mean           0.188588          0.833905               0.205746\n",
       "std            0.391183          2.503083               0.404247\n",
       "min            0.000000        -82.028472               0.000000\n",
       "25%            0.000000          0.045139               0.000000\n",
       "50%            0.000000          0.154861               0.000000\n",
       "75%            0.000000          0.422917               0.000000\n",
       "max            1.000000        175.627778               1.000000"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "elective_surgery         0\n",
       "icu_stay_type            0\n",
       "icu_type                 0\n",
       "pre_icu_los_days         0\n",
       "apache_post_operative    0\n",
       "hospital_traffic_bin     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>icu_stay_type</th>\n",
       "      <th>icu_type</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>apache_post_operative</th>\n",
       "      <th>hospital_traffic_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>admit</td>\n",
       "      <td>CTICU</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.927778</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>admit</td>\n",
       "      <td>CTICU</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>admit</td>\n",
       "      <td>Med-Surg ICU</td>\n",
       "      <td>0.073611</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   elective_surgery icu_stay_type      icu_type  pre_icu_los_days  \\\n",
       "0                 0         admit         CTICU          0.541667   \n",
       "1                 0         admit  Med-Surg ICU          0.927778   \n",
       "2                 0         admit  Med-Surg ICU          0.000694   \n",
       "3                 1         admit         CTICU          0.000694   \n",
       "4                 0         admit  Med-Surg ICU          0.073611   \n",
       "\n",
       "   apache_post_operative hospital_traffic_bin  \n",
       "0                      0                    5  \n",
       "1                      0                    1  \n",
       "2                      0                    5  \n",
       "3                      1                    5  \n",
       "4                      0                    1  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "admit       125263\n",
       "transfer      5140\n",
       "readmit        618\n",
       "Name: icu_stay_type, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering['icu_stay_type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Med-Surg ICU    71954\n",
       "CCU-CTICU       11279\n",
       "MICU            11007\n",
       "Neuro ICU       10616\n",
       "Cardiac ICU      8337\n",
       "SICU             8305\n",
       "CSICU            5021\n",
       "CTICU            4502\n",
       "Name: icu_type, dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering['icu_type'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Expand Categorical Features into Dummy Variables\n",
    "* 'icu_stay_type' - 3 categories\n",
    "* 'icu_type' - 8 categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['elective_surgery', 'pre_icu_los_days', 'apache_post_operative',\n",
       "       'hospital_traffic_bin', 'icu_stay_type_readmit',\n",
       "       'icu_stay_type_transfer', 'icu_type_CSICU', 'icu_type_CTICU',\n",
       "       'icu_type_Cardiac ICU', 'icu_type_MICU', 'icu_type_Med-Surg ICU',\n",
       "       'icu_type_Neuro ICU', 'icu_type_SICU'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To select all columns leave the columns keyword empty; defaults to all  \n",
    "df_clustering = pd.get_dummies(df_clustering, columns=['icu_stay_type','icu_type'], drop_first=True)\n",
    "df_clustering.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>apache_post_operative</th>\n",
       "      <th>hospital_traffic_bin</th>\n",
       "      <th>icu_stay_type_readmit</th>\n",
       "      <th>icu_stay_type_transfer</th>\n",
       "      <th>icu_type_CSICU</th>\n",
       "      <th>icu_type_CTICU</th>\n",
       "      <th>icu_type_Cardiac ICU</th>\n",
       "      <th>icu_type_MICU</th>\n",
       "      <th>icu_type_Med-Surg ICU</th>\n",
       "      <th>icu_type_Neuro ICU</th>\n",
       "      <th>icu_type_SICU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.927778</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.073611</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   elective_surgery  pre_icu_los_days  apache_post_operative  \\\n",
       "0                 0          0.541667                      0   \n",
       "1                 0          0.927778                      0   \n",
       "2                 0          0.000694                      0   \n",
       "3                 1          0.000694                      1   \n",
       "4                 0          0.073611                      0   \n",
       "\n",
       "  hospital_traffic_bin  icu_stay_type_readmit  icu_stay_type_transfer  \\\n",
       "0                    5                      0                       0   \n",
       "1                    1                      0                       0   \n",
       "2                    5                      0                       0   \n",
       "3                    5                      0                       0   \n",
       "4                    1                      0                       0   \n",
       "\n",
       "   icu_type_CSICU  icu_type_CTICU  icu_type_Cardiac ICU  icu_type_MICU  \\\n",
       "0               0               1                     0              0   \n",
       "1               0               0                     0              0   \n",
       "2               0               0                     0              0   \n",
       "3               0               1                     0              0   \n",
       "4               0               0                     0              0   \n",
       "\n",
       "   icu_type_Med-Surg ICU  icu_type_Neuro ICU  icu_type_SICU  \n",
       "0                      0                   0              0  \n",
       "1                      1                   0              0  \n",
       "2                      1                   0              0  \n",
       "3                      0                   0              0  \n",
       "4                      1                   0              0  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Cluster\n",
    "\n",
    "**Training and test data used as one dataset**\n",
    "\n",
    "The silhouette value is a measure of how similar an object is to its own cluster (cohesion) compared to other clusters (separation). The silhouette ranges from −1 to +1, where a **high value indicates that the object is well matched** to its own cluster and poorly matched to neighboring clusters. - Wikipedia "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N: 3, Score: 0.4045155382287674\n",
      "N: 5, Score: 0.29925682559484185\n",
      "N: 7, Score: 0.3042943664783366\n",
      "N: 9, Score: 0.3308217203601151\n",
      "N: 11, Score: 0.31985016284539336\n",
      "N: 13, Score: 0.32039096410303264\n",
      "N: 15, Score: 0.32241385307418646\n",
      "N: 17, Score: 0.3532676169842306\n",
      "N: 19, Score: 0.38538236512668633\n",
      "N: 21, Score: 0.4001258557680569\n",
      "N: 23, Score: 0.40333970651207635\n",
      "N: 25, Score: 0.4145221953736163\n",
      "N: 27, Score: 0.43167483467339596\n",
      "N: 29, Score: 0.4320987612386619\n"
     ]
    }
   ],
   "source": [
    "k_scores = {}\n",
    "for n in range(3,30,2):\n",
    "    num_clusters = n\n",
    "    km = KMeans(n_clusters=num_clusters,random_state=10,n_init=1) # n_init, number of times the K-mean algorithm will run\n",
    "    km.fit(df_clustering)\n",
    "    cluster_labels = km.fit_predict(df_clustering)\n",
    "    cluster_labels\n",
    "    silhouette_avg = silhouette_score(df_clustering, cluster_labels)\n",
    "    k_scores['n'] = silhouette_avg\n",
    "    print(\"N: \" + str(n) + \", Score: \" + str(silhouette_avg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "started running at 3:28pm. Finished 4:55pm. Wayyy too long. :/ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N: 27, Score: 0.43167483467339596\n",
      "N: 31, Score: 0.431825140712791\n",
      "N: 35, Score: 0.44956554514712466\n"
     ]
    }
   ],
   "source": [
    "for n in range(27,38,4):\n",
    "    num_clusters = n\n",
    "    km = KMeans(n_clusters=num_clusters,random_state=10,n_init=1) # n_init, number of times the K-mean algorithm will run\n",
    "    km.fit(df_clustering)\n",
    "    cluster_labels = km.fit_predict(df_clustering)\n",
    "    cluster_labels\n",
    "    silhouette_avg = silhouette_score(df_clustering, cluster_labels)\n",
    "    k_scores['n'] = silhouette_avg\n",
    "    print(\"N: \" + str(n) + \", Score: \" + str(silhouette_avg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N: 35, Score: 0.44956554514712466\n",
      "N: 39, Score: 0.47819695576839255\n",
      "N: 43, Score: 0.48611635453484286\n",
      "N: 47, Score: 0.5152354192460147\n"
     ]
    }
   ],
   "source": [
    "for n in range(35,51,4):\n",
    "    num_clusters = n\n",
    "    km = KMeans(n_clusters=num_clusters,random_state=10,n_init=1) # n_init, number of times the K-mean algorithm will run\n",
    "    km.fit(df_clustering)\n",
    "    cluster_labels = km.fit_predict(df_clustering)\n",
    "    cluster_labels\n",
    "    silhouette_avg = silhouette_score(df_clustering, cluster_labels)\n",
    "    k_scores['n'] = silhouette_avg\n",
    "    print(\"N: \" + str(n) + \", Score: \" + str(silhouette_avg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choosing 9 clusters. Don't want too many to have to use to fill in. \n",
    "Stands out a little more than the others. But mostly an arbitrary distinction. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,\n",
       "       n_clusters=9, n_init=1, n_jobs=None, precompute_distances='auto',\n",
       "       random_state=10, tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_clusters = 9\n",
    "km = KMeans(n_clusters=num_clusters,random_state=10,n_init=1) # n_init, number of times the K-mean algorithm will run\n",
    "km.fit(df_clustering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add labels to the dataframe \n",
    "df_clustering['clusters'] = km.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>elective_surgery</th>\n",
       "      <th>pre_icu_los_days</th>\n",
       "      <th>apache_post_operative</th>\n",
       "      <th>hospital_traffic_bin</th>\n",
       "      <th>icu_stay_type_readmit</th>\n",
       "      <th>icu_stay_type_transfer</th>\n",
       "      <th>icu_type_CSICU</th>\n",
       "      <th>icu_type_CTICU</th>\n",
       "      <th>icu_type_Cardiac ICU</th>\n",
       "      <th>icu_type_MICU</th>\n",
       "      <th>icu_type_Med-Surg ICU</th>\n",
       "      <th>icu_type_Neuro ICU</th>\n",
       "      <th>icu_type_SICU</th>\n",
       "      <th>clusters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.927778</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000694</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.073611</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   elective_surgery  pre_icu_los_days  apache_post_operative  \\\n",
       "0                 0          0.541667                      0   \n",
       "1                 0          0.927778                      0   \n",
       "2                 0          0.000694                      0   \n",
       "3                 1          0.000694                      1   \n",
       "4                 0          0.073611                      0   \n",
       "\n",
       "  hospital_traffic_bin  icu_stay_type_readmit  icu_stay_type_transfer  \\\n",
       "0                    5                      0                       0   \n",
       "1                    1                      0                       0   \n",
       "2                    5                      0                       0   \n",
       "3                    5                      0                       0   \n",
       "4                    1                      0                       0   \n",
       "\n",
       "   icu_type_CSICU  icu_type_CTICU  icu_type_Cardiac ICU  icu_type_MICU  \\\n",
       "0               0               1                     0              0   \n",
       "1               0               0                     0              0   \n",
       "2               0               0                     0              0   \n",
       "3               0               1                     0              0   \n",
       "4               0               0                     0              0   \n",
       "\n",
       "   icu_type_Med-Surg ICU  icu_type_Neuro ICU  icu_type_SICU  clusters  \n",
       "0                      0                   0              0         1  \n",
       "1                      1                   0              0         0  \n",
       "2                      1                   0              0         1  \n",
       "3                      0                   0              0         1  \n",
       "4                      1                   0              0         0  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    50169\n",
       "1    32357\n",
       "7    19721\n",
       "8    14464\n",
       "4     8701\n",
       "6     4018\n",
       "2     1280\n",
       "3      279\n",
       "5       32\n",
       "Name: clusters, dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clustering['clusters'].value_counts()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Save dataframe with labels \n",
    "with open('df_demographic_clusters', 'wb') as to_write:\n",
    "    pickle.dump(df_clustering, to_write) "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Save dictionary with silhouette scores\n",
    "with open('silhouette_values', 'wb') as to_write:\n",
    "    pickle.dump(k_scores, to_write) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n': 0.5152354192460147}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Silhouette Scores\n",
    "N: 3, Score: 0.4045155382287674\n",
    "N: 5, Score: 0.29925682559484185\n",
    "N: 7, Score: 0.3042943664783366\n",
    "N: 9, Score: 0.3308217203601151\n",
    "N: 11, Score: 0.31985016284539336\n",
    "N: 13, Score: 0.32039096410303264\n",
    "N: 15, Score: 0.32241385307418646\n",
    "N: 17, Score: 0.3532676169842306\n",
    "N: 19, Score: 0.38538236512668633\n",
    "N: 21, Score: 0.4001258557680569\n",
    "N: 23, Score: 0.40333970651207635\n",
    "N: 25, Score: 0.4145221953736163\n",
    "N: 27, Score: 0.43167483467339596\n",
    "N: 29, Score: 0.4320987612386619\n",
    "\n",
    "N: 27, Score: 0.43167483467339596\n",
    "N: 31, Score: 0.431825140712791\n",
    "N: 35, Score: 0.44956554514712466\n",
    "\n",
    "N: 35, Score: 0.44956554514712466\n",
    "N: 39, Score: 0.47819695576839255\n",
    "N: 43, Score: 0.48611635453484286\n",
    "N: 47, Score: 0.5152354192460147"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Didn't add to dictionary properly. Adding it now for purposes of graphing. \n",
    "# Not sure it's worth graphing. Will come back to if we decide to use these. \n",
    "\n",
    "k_scores = {3:.4045155382287674, \n",
    "            5: .29925682559484185,\n",
    "            7:.3042943664783366\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{3: 0.4045155382287674, 5: 0.29925682559484185, 7: 0.3042943664783366}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary:\n",
    "Hmmm I'm not sure these clusters are cohesive enough to really base decisons on filling in missing data for. \n",
    "\n",
    "We'll try moving over to PPCA and see if that works better for filling in the missing data. \n",
    "\n",
    "It does mean this model will not be interpretable, however. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}