xray_covid_baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,sys\n",
    "sys.path.insert(0,\"../torchxrayvision/\")\n",
    "from glob import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import sys\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchxrayvision as xrv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_model = xrv.models.DenseNet(weights=\"all\")#.cuda()\n",
    "nn_model.op_threshs = None # to prevent sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision, torchvision.transforms\n",
    "transform = torchvision.transforms.Compose([xrv.datasets.XRayCenterCrop(),xrv.datasets.XRayResizer(224)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data is at `git checkout 8e7e5900032e0eacbfee0e604561ea3a258e2b2a`\n",
    "# https://github.com/ieee8023/covid-chestxray-dataset\n",
    "path_to_covid_dataset = \"../covid-chestxray-dataset\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge = xrv.datasets.COVID19_Dataset(imgpath=path_to_covid_dataset + \"/images\",\n",
    "                                      csvpath=path_to_covid_dataset + \"/metadata.csv\",\n",
    "                                      transform=transform, views=[\"PA\", \"AP\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv = dmerge.csv.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Aspergillosis': {0.0: 517, 1.0: 1},\n",
      " 'Aspiration': {0.0: 517, 1.0: 1},\n",
      " 'Bacterial': {0.0: 472, 1.0: 46},\n",
      " 'COVID-19': {0.0: 186, 1.0: 332},\n",
      " 'Chlamydophila': {0.0: 517, 1.0: 1},\n",
      " 'Fungal': {0.0: 495, 1.0: 23},\n",
      " 'H1N1': {0.0: 517, 1.0: 1},\n",
      " 'Herpes ': {0.0: 515, 1.0: 3},\n",
      " 'Influenza': {0.0: 514, 1.0: 4},\n",
      " 'Klebsiella': {0.0: 509, 1.0: 9},\n",
      " 'Legionella': {0.0: 511, 1.0: 7},\n",
      " 'Lipoid': {0.0: 510, 1.0: 8},\n",
      " 'MERS-CoV': {0.0: 510, 1.0: 8},\n",
      " 'MRSA': {0.0: 517, 1.0: 1},\n",
      " 'Mycoplasma': {0.0: 513, 1.0: 5},\n",
      " 'No Finding': {0.0: 504, 1.0: 14},\n",
      " 'Nocardia': {0.0: 514, 1.0: 4},\n",
      " 'Pneumocystis': {0.0: 496, 1.0: 22},\n",
      " 'Pneumonia': {0.0: 25, 1.0: 493},\n",
      " 'SARS': {0.0: 502, 1.0: 16},\n",
      " 'Staphylococcus': {0.0: 517, 1.0: 1},\n",
      " 'Streptococcus': {0.0: 501, 1.0: 17},\n",
      " 'Tuberculosis': {0.0: 507, 1.0: 11},\n",
      " 'Varicella': {0.0: 513, 1.0: 5},\n",
      " 'Viral': {0.0: 150, 1.0: 368}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "COVID19_Dataset num_samples=518 views=['PA', 'AP']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dmerge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv = dmerge.csv.join(pd.DataFrame(dmerge.labels, columns=[\"gt_\" + path for path in dmerge.pathologies]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = pd.read_csv(path_to_covid_dataset + \"/annotations/covid-severity-scores.csv\", skiprows=5)\n",
    "scores = scores.set_index(\"filename\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geographic_mean</th>\n",
       "      <th>opacity_mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>01E392EE-69F9-4E33-BFCE-E5C968654078.jpeg</th>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03BF7561-A9BA-4C3C-B8A0-D3E585F73F3C.jpeg</th>\n",
       "      <td>2.7</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-s2.0-S0140673620303706-fx1_lrg.jpg</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-s2.0-S1684118220300608-main.pdf-001.jpg</th>\n",
       "      <td>3.7</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1-s2.0-S1684118220300608-main.pdf-002.jpg</th>\n",
       "      <td>1.7</td>\n",
       "      <td>1.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           geographic_mean  opacity_mean\n",
       "filename                                                                \n",
       "01E392EE-69F9-4E33-BFCE-E5C968654078.jpeg              6.0           4.0\n",
       "03BF7561-A9BA-4C3C-B8A0-D3E585F73F3C.jpeg              2.7           2.0\n",
       "1-s2.0-S0140673620303706-fx1_lrg.jpg                   2.0           2.0\n",
       "1-s2.0-S1684118220300608-main.pdf-001.jpg              3.7           3.0\n",
       "1-s2.0-S1684118220300608-main.pdf-002.jpg              1.7           1.7"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv = dmerge.csv.set_index(\"filename\").join(scores, on=\"filename\").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    90.000000\n",
       "mean      3.280000\n",
       "std       2.332631\n",
       "min       0.000000\n",
       "25%       1.300000\n",
       "50%       3.000000\n",
       "75%       4.700000\n",
       "max       8.000000\n",
       "Name: geographic_mean, dtype: float64"
      ]
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dmerge.csv.geographic_mean[~dmerge.csv.geographic_mean.isna()].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dl = torch.utils.data.DataLoader(dmerge, batch_size=32, num_workers=2, pin_memory=True)\n",
    "# d_feats = []\n",
    "# d_preds = []\n",
    "# d_pixels = []\n",
    "# with torch.no_grad():\n",
    "#     for i, batch in enumerate(dl):\n",
    "#         imgs = batch[\"img\"]#.cuda()\n",
    "#         out = nn_model.features(imgs)\n",
    "#         out = F.relu(out, inplace=True)\n",
    "#         out = F.adaptive_avg_pool2d(out, (1, 1))\n",
    "#         pred = nn_model.classifier(out.view(out.size(0), -1))\n",
    "#         d_pixels.append(imgs)\n",
    "#         d_feats.append(out.detach().cpu().numpy())\n",
    "#         d_preds.append(pred.detach().cpu().numpy())\n",
    "#         if i %10 == 0:\n",
    "#             print(i)\n",
    "# d_feats = np.concatenate(d_feats).reshape(-1,1024)\n",
    "# d_preds = np.concatenate(d_preds)\n",
    "# d_pixels = np.concatenate(d_pixels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "#pickle.dump([d_feats, d_preds, d_pixels], open(\"d_feats2.pkl\",\"wb\"))\n",
    "d_feats, d_preds, d_pixels = pickle.load(open(\"d_feats2.pkl\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(518, 1024)"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv = dmerge.csv.join(pd.DataFrame(d_preds, columns=xrv.datasets.default_pathologies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv[\"country\"] = dmerge.csv.location.str.split(\",\").str[-1].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "continents = pd.read_csv(\"continents.csv.zip\")\n",
    "continents = continents[[\"name\", \"region\", \"sub-region\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv = dmerge.csv.join(continents.set_index(\"name\"), on=[\"country\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: country, dtype: object)"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# should be empty (it not then change the country name)\n",
    "dmerge.csv [(~dmerge.csv.country.isna() & dmerge.csv.region.isna())].country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(518, 18)"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(518, 1024)"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv.country = dmerge.csv.country.fillna(\"Unknown\")\n",
    "dmerge.csv.region = dmerge.csv.region.fillna(\"Unknown\")\n",
    "dmerge.csv[\"sub-region\"] = dmerge.csv[\"sub-region\"].fillna(\"Unknown\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "dmerge.csv[\"Frontal\"] = dmerge.csv.view.isin([\"AP\", \"PA\", \"AP Supine\"])\n",
    "dmerge.csv[\"Lateral\"] = dmerge.csv.view.isin([\"L\"])\n",
    "dmerge.csv[\"SurvivalY\"] = dmerge.csv.survival.isin([\"Y\"])\n",
    "dmerge.csv[\"SurvivalN\"] = dmerge.csv.survival.isin([\"N\"])\n",
    "dmerge.csv[\"Went ICUY\"] = dmerge.csv.went_icu.isin([\"Y\"])\n",
    "dmerge.csv[\"Went ICUN\"] = dmerge.csv.went_icu.isin([\"N\"])\n",
    "dmerge.csv[\"IntubationY\"] = dmerge.csv.intubated.isin([\"Y\"])\n",
    "dmerge.csv[\"IntubationN\"] = dmerge.csv.intubated.isin([\"N\"])\n",
    "dmerge.csv[\"COVID-19\"] = dmerge.csv[\"finding\"].str.contains(\"COVID-19\")\n",
    "dmerge.csv[\"RT-PCR Positive\"] = dmerge.csv[\"RT_PCR_positive\"].isin([\"Y\"])\n",
    "dmerge.csv[\"Male Ratio\"] = dmerge.csv.sex == \"M\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "age           54.231618\n",
       "Male Ratio     0.551320\n",
       "dtype: float64"
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dmerge.csv.groupby(\"patientid\").first()[[\"age\",\"Male Ratio\"]].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>age</th>\n",
       "      <th>COVID-19</th>\n",
       "      <th>RT-PCR Positive</th>\n",
       "      <th>Male Ratio</th>\n",
       "      <th>Survival Y/N</th>\n",
       "      <th>Went ICU Y/N</th>\n",
       "      <th>Intubation Y/N</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>region</th>\n",
       "      <th>sub-region</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Africa</th>\n",
       "      <th>Northern Africa</th>\n",
       "      <td>5</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0/1</td>\n",
       "      <td>0/0</td>\n",
       "      <td>0/0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Americas</th>\n",
       "      <th>Latin America and the Caribbean</th>\n",
       "      <td>3</td>\n",
       "      <td>44.67</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.67</td>\n",
       "      <td>1/0</td>\n",
       "      <td>0/1</td>\n",
       "      <td>0/1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Northern America</th>\n",
       "      <td>21</td>\n",
       "      <td>50.30</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>0.57</td>\n",
       "      <td>5/4</td>\n",
       "      <td>7/1</td>\n",
       "      <td>7/3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Asia</th>\n",
       "      <th>Eastern Asia</th>\n",
       "      <td>48</td>\n",
       "      <td>54.32</td>\n",
       "      <td>45</td>\n",
       "      <td>35</td>\n",
       "      <td>0.31</td>\n",
       "      <td>16/3</td>\n",
       "      <td>3/5</td>\n",
       "      <td>5/6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>South-eastern Asia</th>\n",
       "      <td>9</td>\n",
       "      <td>46.67</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.56</td>\n",
       "      <td>4/1</td>\n",
       "      <td>1/3</td>\n",
       "      <td>1/3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Southern Asia</th>\n",
       "      <td>14</td>\n",
       "      <td>48.86</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>0.71</td>\n",
       "      <td>4/0</td>\n",
       "      <td>3/0</td>\n",
       "      <td>3/0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western Asia</th>\n",
       "      <td>9</td>\n",
       "      <td>41.00</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0.78</td>\n",
       "      <td>2/0</td>\n",
       "      <td>2/0</td>\n",
       "      <td>1/0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Europe</th>\n",
       "      <th>Eastern Europe</th>\n",
       "      <td>2</td>\n",
       "      <td>60.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0/0</td>\n",
       "      <td>0/0</td>\n",
       "      <td>0/0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Northern Europe</th>\n",
       "      <td>30</td>\n",
       "      <td>59.88</td>\n",
       "      <td>21</td>\n",
       "      <td>17</td>\n",
       "      <td>0.57</td>\n",
       "      <td>3/3</td>\n",
       "      <td>7/0</td>\n",
       "      <td>7/1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Southern Europe</th>\n",
       "      <td>99</td>\n",
       "      <td>57.83</td>\n",
       "      <td>67</td>\n",
       "      <td>54</td>\n",
       "      <td>0.61</td>\n",
       "      <td>27/2</td>\n",
       "      <td>18/9</td>\n",
       "      <td>9/21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western Europe</th>\n",
       "      <td>34</td>\n",
       "      <td>56.40</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>0.59</td>\n",
       "      <td>8/0</td>\n",
       "      <td>7/23</td>\n",
       "      <td>1/0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oceania</th>\n",
       "      <th>Australia and New Zealand</th>\n",
       "      <td>39</td>\n",
       "      <td>50.65</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.54</td>\n",
       "      <td>4/0</td>\n",
       "      <td>1/1</td>\n",
       "      <td>1/1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unknown</th>\n",
       "      <th>Unknown</th>\n",
       "      <td>28</td>\n",
       "      <td>54.16</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.61</td>\n",
       "      <td>3/2</td>\n",
       "      <td>5/0</td>\n",
       "      <td>6/2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         filename    age COVID-19  \\\n",
       "                                            count   mean      sum   \n",
       "region   sub-region                                                 \n",
       "Africa   Northern Africa                        5  40.00        0   \n",
       "Americas Latin America and the Caribbean        3  44.67        1   \n",
       "         Northern America                      21  50.30       11   \n",
       "Asia     Eastern Asia                          48  54.32       45   \n",
       "         South-eastern Asia                     9  46.67        8   \n",
       "         Southern Asia                         14  48.86       13   \n",
       "         Western Asia                           9  41.00        4   \n",
       "Europe   Eastern Europe                         2  60.00        0   \n",
       "         Northern Europe                       30  59.88       21   \n",
       "         Southern Europe                       99  57.83       67   \n",
       "         Western Europe                        34  56.40       30   \n",
       "Oceania  Australia and New Zealand             39  50.65        1   \n",
       "Unknown  Unknown                               28  54.16       18   \n",
       "\n",
       "                                         RT-PCR Positive Male Ratio  \\\n",
       "                                                     sum       mean   \n",
       "region   sub-region                                                   \n",
       "Africa   Northern Africa                               0       0.00   \n",
       "Americas Latin America and the Caribbean               1       0.67   \n",
       "         Northern America                             10       0.57   \n",
       "Asia     Eastern Asia                                 35       0.31   \n",
       "         South-eastern Asia                            8       0.56   \n",
       "         Southern Asia                                13       0.71   \n",
       "         Western Asia                                  2       0.78   \n",
       "Europe   Eastern Europe                                0       1.00   \n",
       "         Northern Europe                              17       0.57   \n",
       "         Southern Europe                              54       0.61   \n",
       "         Western Europe                                4       0.59   \n",
       "Oceania  Australia and New Zealand                     2       0.54   \n",
       "Unknown  Unknown                                       0       0.61   \n",
       "\n",
       "                                         Survival Y/N Went ICU Y/N  \\\n",
       "                                                                     \n",
       "region   sub-region                                                  \n",
       "Africa   Northern Africa                          0/1          0/0   \n",
       "Americas Latin America and the Caribbean          1/0          0/1   \n",
       "         Northern America                         5/4          7/1   \n",
       "Asia     Eastern Asia                            16/3          3/5   \n",
       "         South-eastern Asia                       4/1          1/3   \n",
       "         Southern Asia                            4/0          3/0   \n",
       "         Western Asia                             2/0          2/0   \n",
       "Europe   Eastern Europe                           0/0          0/0   \n",
       "         Northern Europe                          3/3          7/0   \n",
       "         Southern Europe                         27/2         18/9   \n",
       "         Western Europe                           8/0         7/23   \n",
       "Oceania  Australia and New Zealand                4/0          1/1   \n",
       "Unknown  Unknown                                  3/2          5/0   \n",
       "\n",
       "                                         Intubation Y/N  \n",
       "                                                         \n",
       "region   sub-region                                      \n",
       "Africa   Northern Africa                            0/0  \n",
       "Americas Latin America and the Caribbean            0/1  \n",
       "         Northern America                           7/3  \n",
       "Asia     Eastern Asia                               5/6  \n",
       "         South-eastern Asia                         1/3  \n",
       "         Southern Asia                              3/0  \n",
       "         Western Asia                               1/0  \n",
       "Europe   Eastern Europe                             0/0  \n",
       "         Northern Europe                            7/1  \n",
       "         Southern Europe                           9/21  \n",
       "         Western Europe                             1/0  \n",
       "Oceania  Australia and New Zealand                  1/1  \n",
       "Unknown  Unknown                                    6/2  "
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = dmerge.csv.groupby(\"patientid\").first().groupby([\"region\", \"sub-region\"]).agg({\n",
    "                                                         'filename':[\"count\"],\n",
    "                                                         'age':['mean'],\n",
    "                                                         'SurvivalY':['sum'],\n",
    "                                                         'SurvivalN':['sum'],\n",
    "                                                         'Went ICUY':['sum'],\n",
    "                                                         'Went ICUN':['sum'],\n",
    "                                                         'IntubationY':['sum'],\n",
    "                                                         'IntubationN':['sum'],\n",
    "                                                         'COVID-19':['sum'],\n",
    "                                                         'RT-PCR Positive':[\"sum\"],\n",
    "                                                         'Male Ratio':['mean'],\n",
    "                                                        }).round(2)\n",
    "def merge(df,a):\n",
    "    df[a + \" Y/N\"] = df[a + 'Y',\"sum\"].astype(\"str\") + \"/\" + df[a + 'N',\"sum\"].astype(\"str\")\n",
    "    del df[a + 'Y']\n",
    "    del df[a + 'N']\n",
    "    \n",
    "merge(r,\"Survival\")\n",
    "merge(r,\"Went ICU\")\n",
    "merge(r,\"Intubation\")\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(r.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = dmerge.csv.groupby([\"view\"]).agg({\n",
    "                                                         'filename':[\"count\"],\n",
    "                                                         'age':['mean'],\n",
    "                                                         'SurvivalY':['sum'],\n",
    "                                                         'SurvivalN':['sum'],\n",
    "                                                         'Went ICUY':['sum'],\n",
    "                                                         'Went ICUN':['sum'],\n",
    "                                                         'IntubationY':['sum'],\n",
    "                                                         'IntubationN':['sum'],\n",
    "                                                         'COVID-19':['sum'],\n",
    "                                                         'RT-PCR Positive':[\"sum\"],\n",
    "                                                         'Male Ratio':['mean'],\n",
    "                                                        }).round(2)\n",
    "\n",
    "def merge(df,a):\n",
    "    df[a + \" Y/N\"] = df[a + 'Y',\"sum\"].astype(\"str\") + \"/\" + df[a + 'N',\"sum\"].astype(\"str\")\n",
    "    del df[a + 'Y']\n",
    "    del df[a + 'N']\n",
    "    \n",
    "merge(r,\"Survival\")\n",
    "merge(r,\"Went ICU\")\n",
    "merge(r,\"Intubation\")\n",
    "#r[\"Survival Y/N\"] = r['SurvivalY',\"sum\"].astype(\"str\") + \"/\" + r['SurvivalN',\"sum\"].astype(\"str\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>age</th>\n",
       "      <th>COVID-19</th>\n",
       "      <th>RT-PCR Positive</th>\n",
       "      <th>Male Ratio</th>\n",
       "      <th>Survival Y/N</th>\n",
       "      <th>Went ICU Y/N</th>\n",
       "      <th>Intubation Y/N</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>view</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AP</th>\n",
       "      <td>192</td>\n",
       "      <td>57.93</td>\n",
       "      <td>141</td>\n",
       "      <td>109</td>\n",
       "      <td>0.54</td>\n",
       "      <td>36/13</td>\n",
       "      <td>58/6</td>\n",
       "      <td>51/15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PA</th>\n",
       "      <td>326</td>\n",
       "      <td>52.21</td>\n",
       "      <td>191</td>\n",
       "      <td>112</td>\n",
       "      <td>0.54</td>\n",
       "      <td>104/19</td>\n",
       "      <td>53/67</td>\n",
       "      <td>33/55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     filename    age COVID-19 RT-PCR Positive Male Ratio Survival Y/N  \\\n",
       "        count   mean      sum             sum       mean                \n",
       "view                                                                    \n",
       "AP        192  57.93      141             109       0.54        36/13   \n",
       "PA        326  52.21      191             112       0.54       104/19   \n",
       "\n",
       "     Went ICU Y/N Intubation Y/N  \n",
       "                                  \n",
       "view                              \n",
       "AP           58/6          51/15  \n",
       "PA          53/67          33/55  "
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(r.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [],
   "source": [
    "#specify task\n",
    "task = \"survival\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "if task==\"geographic-extent\":\n",
    "    print(task)\n",
    "    target_str = \"Geographic Extent\"\n",
    "    labels_mask = ~np.isnan(dmerge.csv.geographic_mean.astype(float))\n",
    "    labels = dmerge.csv.geographic_mean[labels_mask]\n",
    "    method = \"linear\"\n",
    "\n",
    "if task==\"opacity\":\n",
    "    print(task)\n",
    "    target_str = \"Opacity\"\n",
    "    labels_mask = ~np.isnan(dmerge.csv.opacity_mean.astype(float))\n",
    "    labels = dmerge.csv.opacity_mean[labels_mask]\n",
    "    method = \"linear\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "if task==\"covid19\":\n",
    "    print(task)\n",
    "    target_str = \"COVID-19\"\n",
    "    labels_mask = ~dmerge.csv.finding.str.contains(\"todo\") & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") > 0) & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") < 8) \n",
    "    labels = dmerge.csv.finding.str.contains(\"COVID-19\")[labels_mask]\n",
    "    method = \"logistic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "if task==\"viral-bacterial\":\n",
    "    print(task)\n",
    "    target_str = \"Viral or Bacterial\"\n",
    "    labels_mask = ((dmerge.csv.gt_Bacterial == 1) | (dmerge.csv.gt_Viral == 1))& \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") > 0) & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") < 8) \n",
    "    labels = (dmerge.csv.gt_Viral == 1)[labels_mask]\n",
    "    method = \"logistic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "survival\n"
     ]
    }
   ],
   "source": [
    "if task==\"survival\":\n",
    "    print(task)\n",
    "    target_str = \"Survival\"\n",
    "    labels_mask = dmerge.csv.survival.isin([\"Y\",\"N\"]) & \\\n",
    "                  ~(dmerge.csv.intubation_present == \"Y\") & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") > 0) & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") < 8)\n",
    "    labels = dmerge.csv.survival[labels_mask] == \"Y\"\n",
    "    method = \"logistic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "if task==\"icu-stay\":\n",
    "    print(task)\n",
    "    target_str = \"ICU Stay\"\n",
    "    labels_mask = dmerge.csv.went_icu.isin([\"Y\",\"N\"]) & \\\n",
    "                  ~(dmerge.csv.in_icu == \"Y\") & \\\n",
    "                  ~(dmerge.csv.intubation_present == \"Y\") & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") > 0) & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") < 8)\n",
    "    labels = (dmerge.csv.went_icu[labels_mask] == \"Y\")[labels_mask]\n",
    "    method = \"logistic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [],
   "source": [
    "if task==\"intubated\":\n",
    "    print(task)\n",
    "    target_str = \"Intubated\"\n",
    "    labels_mask = dmerge.csv.intubated.isin([\"Y\",\"N\"]) & \\\n",
    "                  ~(dmerge.csv.intubation_present == \"Y\") & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") > 0) & \\\n",
    "                  (dmerge.csv.offset.astype(\"float\") < 8)\n",
    "    labels = (dmerge.csv.intubated[labels_mask] == \"Y\")[labels_mask]\n",
    "    method = \"logistic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_mask.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Asia', 'Americas', 'Europe', 'Oceania', 'Unknown'], dtype=object)"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dmerge.csv.region[labels_mask].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['South-eastern Asia', 'Northern America', 'Eastern Asia',\n",
       "       'Southern Europe', 'Australia and New Zealand', 'Northern Europe',\n",
       "       'Unknown', 'Western Europe', 'Latin America and the Caribbean',\n",
       "       'Southern Asia'], dtype=object)"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dmerge.csv[\"sub-region\"][labels_mask].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [],
   "source": [
    "groupby = \"region\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Asia {True: 20, False: 3}\n",
      "Americas {True: 4, False: 1}\n",
      "Europe {True: 19, False: 3}\n",
      "Oceania {True: 1}\n",
      "Unknown {True: 2}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Asia', 'Americas', 'Europe']"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_regions = []\n",
    "if method == \"logistic\":\n",
    "    for region in dmerge.csv[groupby][labels_mask].dropna().unique():\n",
    "        counts = labels[labels_mask][dmerge.csv[groupby][labels_mask] == region].value_counts()\n",
    "        print(region, dict(counts))\n",
    "        if len(counts) > 1 and counts.min() > 0:\n",
    "            test_regions.append(region)        \n",
    "if method == \"linear\":\n",
    "    for region in dmerge.csv[groupby][labels_mask].dropna().unique():\n",
    "        if (dmerge.csv[groupby][labels_mask] == region).sum() > 2:\n",
    "            test_regions.append(region)    \n",
    "\n",
    "if \"Unknown\" in test_regions:\n",
    "    test_regions.remove(\"Unknown\")\n",
    "test_regions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data = d_feats[labels_mask]\n",
    "for test_region in test_regions:\n",
    "    results = results.append(utils.evaluate(data, labels, \"Intermediate features\", plot=False, \n",
    "                                      regions = dmerge.csv[groupby][labels_mask],\n",
    "                                      test_region=test_region,\n",
    "                                      target_str=target_str, \n",
    "                                      method=method), ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.zeros((data.shape[0],1))\n",
    "for test_region in test_regions:\n",
    "    results = results.append(utils.evaluate(data, labels, \"Baseline prevalence\", plot=False, \n",
    "                                      regions = dmerge.csv[groupby][labels_mask],\n",
    "                                      test_region=test_region,\n",
    "                                      target_str=target_str, \n",
    "                                      method=method), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = d_preds[labels_mask]\n",
    "for test_region in test_regions:\n",
    "    results = results.append(utils.evaluate(data, labels, \"18 outputs\", plot=False, \n",
    "                                      regions = dmerge.csv[groupby][labels_mask],\n",
    "                                      test_region=test_region,\n",
    "                                      target_str=target_str, \n",
    "                                      method=method), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = d_preds[labels_mask]\n",
    "data = data[:,[xrv.datasets.default_pathologies.index(\"Lung Opacity\"),\n",
    "               xrv.datasets.default_pathologies.index(\"Pneumonia\"),\n",
    "               xrv.datasets.default_pathologies.index(\"Infiltration\"),\n",
    "               xrv.datasets.default_pathologies.index(\"Consolidation\")]]\n",
    "\n",
    "for test_region in test_regions:\n",
    "    results = results.append(utils.evaluate(data, labels, \"4 outputs\", plot=False, \n",
    "                                      regions = dmerge.csv[groupby][labels_mask],\n",
    "                                      test_region=test_region,\n",
    "                                      target_str=target_str, \n",
    "                                      method=method), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data = d_preds[labels_mask]\n",
    "data = data[:,[xrv.datasets.default_pathologies.index(\"Lung Opacity\")]]\n",
    "for test_region in test_regions:\n",
    "    results = results.append(utils.evaluate(data, labels, \"lung opacity output\", plot=False, \n",
    "                                      regions = dmerge.csv[groupby][labels_mask],\n",
    "                                      test_region=test_region,\n",
    "                                      target_str=target_str, \n",
    "                                      method=method), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = d_pixels.reshape(d_pixels.shape[0],-1)[labels_mask]\n",
    "# for test_region in test_regions:\n",
    "#     for seed in range(3):\n",
    "#         results = results.append(utils.evaluate(data, labels, \"Image pixels\", plot=False, \n",
    "#                                           regions = dmerge.csv[groupby][labels_mask],\n",
    "#                                           test_region=test_region,\n",
    "#                                           target_str=target_str, \n",
    "#                                           method=method, \n",
    "#                                           seed=seed,\n",
    "#                                           usemlp=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># params</th>\n",
       "      <th># test samples</th>\n",
       "      <th>AUPRC</th>\n",
       "      <th>AUROC</th>\n",
       "      <th>method</th>\n",
       "      <th>name</th>\n",
       "      <th>test_region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1024+1</td>\n",
       "      <td>{True: 20, False: 3}</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Intermediate features</td>\n",
       "      <td>Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1024+1</td>\n",
       "      <td>{False: 1, True: 4}</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Intermediate features</td>\n",
       "      <td>Americas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1024+1</td>\n",
       "      <td>{True: 19, False: 3}</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Intermediate features</td>\n",
       "      <td>Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{True: 20, False: 3}</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Baseline prevalence</td>\n",
       "      <td>Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{False: 1, True: 4}</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Baseline prevalence</td>\n",
       "      <td>Americas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{True: 19, False: 3}</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>Baseline prevalence</td>\n",
       "      <td>Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>18+1</td>\n",
       "      <td>{True: 20, False: 3}</td>\n",
       "      <td>0.858385</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>18 outputs</td>\n",
       "      <td>Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>18+1</td>\n",
       "      <td>{False: 1, True: 4}</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>18 outputs</td>\n",
       "      <td>Americas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>18+1</td>\n",
       "      <td>{True: 19, False: 3}</td>\n",
       "      <td>0.851435</td>\n",
       "      <td>0.447368</td>\n",
       "      <td>logistic</td>\n",
       "      <td>18 outputs</td>\n",
       "      <td>Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4+1</td>\n",
       "      <td>{True: 20, False: 3}</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>4 outputs</td>\n",
       "      <td>Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4+1</td>\n",
       "      <td>{False: 1, True: 4}</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>4 outputs</td>\n",
       "      <td>Americas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4+1</td>\n",
       "      <td>{True: 19, False: 3}</td>\n",
       "      <td>0.891463</td>\n",
       "      <td>0.614035</td>\n",
       "      <td>logistic</td>\n",
       "      <td>4 outputs</td>\n",
       "      <td>Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{True: 20, False: 3}</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>lung opacity output</td>\n",
       "      <td>Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{False: 1, True: 4}</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>logistic</td>\n",
       "      <td>lung opacity output</td>\n",
       "      <td>Americas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1+1</td>\n",
       "      <td>{True: 19, False: 3}</td>\n",
       "      <td>0.898086</td>\n",
       "      <td>0.640351</td>\n",
       "      <td>logistic</td>\n",
       "      <td>lung opacity output</td>\n",
       "      <td>Europe</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   # params        # test samples     AUPRC     AUROC    method  \\\n",
       "0    1024+1  {True: 20, False: 3}  0.869565  0.500000  logistic   \n",
       "1    1024+1   {False: 1, True: 4}  0.800000  0.500000  logistic   \n",
       "2    1024+1  {True: 19, False: 3}  0.863636  0.500000  logistic   \n",
       "3       1+1  {True: 20, False: 3}  0.869565  0.500000  logistic   \n",
       "4       1+1   {False: 1, True: 4}  0.800000  0.500000  logistic   \n",
       "5       1+1  {True: 19, False: 3}  0.863636  0.500000  logistic   \n",
       "6      18+1  {True: 20, False: 3}  0.858385  0.450000  logistic   \n",
       "7      18+1   {False: 1, True: 4}  0.800000  0.500000  logistic   \n",
       "8      18+1  {True: 19, False: 3}  0.851435  0.447368  logistic   \n",
       "9       4+1  {True: 20, False: 3}  0.869565  0.500000  logistic   \n",
       "10      4+1   {False: 1, True: 4}  0.800000  0.500000  logistic   \n",
       "11      4+1  {True: 19, False: 3}  0.891463  0.614035  logistic   \n",
       "12      1+1  {True: 20, False: 3}  0.869565  0.500000  logistic   \n",
       "13      1+1   {False: 1, True: 4}  0.800000  0.500000  logistic   \n",
       "14      1+1  {True: 19, False: 3}  0.898086  0.640351  logistic   \n",
       "\n",
       "                     name test_region  \n",
       "0   Intermediate features        Asia  \n",
       "1   Intermediate features    Americas  \n",
       "2   Intermediate features      Europe  \n",
       "3     Baseline prevalence        Asia  \n",
       "4     Baseline prevalence    Americas  \n",
       "5     Baseline prevalence      Europe  \n",
       "6              18 outputs        Asia  \n",
       "7              18 outputs    Americas  \n",
       "8              18 outputs      Europe  \n",
       "9               4 outputs        Asia  \n",
       "10              4 outputs    Americas  \n",
       "11              4 outputs      Europe  \n",
       "12    lung opacity output        Asia  \n",
       "13    lung opacity output    Americas  \n",
       "14    lung opacity output      Europe  "
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(results.round(2).to_latex(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">AUROC</th>\n",
       "      <th colspan=\"2\" halign=\"left\">AUPRC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18 outputs</th>\n",
       "      <th>logistic</th>\n",
       "      <td>0.47</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 outputs</th>\n",
       "      <th>logistic</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Baseline prevalence</th>\n",
       "      <th>logistic</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Intermediate features</th>\n",
       "      <th>logistic</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lung opacity output</th>\n",
       "      <th>logistic</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               AUROC       AUPRC      \n",
       "                                mean   std  mean   std\n",
       "name                  method                          \n",
       "18 outputs            logistic  0.47  0.03  0.84  0.03\n",
       "4 outputs             logistic  0.54  0.07  0.85  0.05\n",
       "Baseline prevalence   logistic  0.50  0.00  0.84  0.04\n",
       "Intermediate features logistic  0.50  0.00  0.84  0.04\n",
       "lung opacity output   logistic  0.55  0.08  0.86  0.05"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if method==\"linear\":\n",
    "    a = results.groupby([\"name\", \"method\"]).agg({\n",
    "                                 'Correlation':['mean','std'],\n",
    "                                 'R^2':['mean','std'],\n",
    "                                 'MAE':['mean','std'],\n",
    "                                # 'MSE':['mean','std']}\n",
    "                                }).round(2)\n",
    "if method==\"logistic\":\n",
    "    a = results.groupby([\"name\", \"method\"]).agg({\n",
    "                                 'AUROC':['mean','std'],\n",
    "                                 'AUPRC':['mean','std']}).round(2)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "results[\" \"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "if method==\"linear\":\n",
    "    a = results.groupby([\" \",\" \",\"name\", \"# params\"]).agg({\n",
    "                             'Correlation':['mean','std'],\n",
    "                             'R^2':['mean','std'],\n",
    "                             'MAE':['mean','std']}).round(2).sort_values((\"MAE\", \"mean\"))\n",
    "if method==\"logistic\":\n",
    "    a = results.groupby([\" \",\" \",\"name\", \"# params\"]).agg({\n",
    "                                 'AUROC':['mean','std'],\n",
    "                                 'AUPRC':['mean','std']}).round(2).sort_values((\"AUROC\", \"mean\"), ascending=False)\n",
    "    \n",
    "\n",
    "for c in a.columns.get_level_values(0).unique():\n",
    "    a[c+\"_\"] =  a[(c, \"mean\")].apply(\"{0:0.2f}\".format) + \"$\\pm$\" + a[(c, \"std\")].apply(\"{0:0.2f}\".format)\n",
    "    del a[c]\n",
    "    a[c] = a[c+\"_\"]\n",
    "    del a[c+\"_\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>AUROC</th>\n",
       "      <th>AUPRC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th># params</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\"></th>\n",
       "      <th rowspan=\"5\" valign=\"top\"></th>\n",
       "      <th>lung opacity output</th>\n",
       "      <th>1+1</th>\n",
       "      <td>0.55$\\pm$0.08</td>\n",
       "      <td>0.86$\\pm$0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 outputs</th>\n",
       "      <th>4+1</th>\n",
       "      <td>0.54$\\pm$0.07</td>\n",
       "      <td>0.85$\\pm$0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Baseline prevalence</th>\n",
       "      <th>1+1</th>\n",
       "      <td>0.50$\\pm$0.00</td>\n",
       "      <td>0.84$\\pm$0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Intermediate features</th>\n",
       "      <th>1024+1</th>\n",
       "      <td>0.50$\\pm$0.00</td>\n",
       "      <td>0.84$\\pm$0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18 outputs</th>\n",
       "      <th>18+1</th>\n",
       "      <td>0.47$\\pm$0.03</td>\n",
       "      <td>0.84$\\pm$0.03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            AUROC          AUPRC\n",
       "                                                                \n",
       "    name                  # params                              \n",
       "    lung opacity output   1+1       0.55$\\pm$0.08  0.86$\\pm$0.05\n",
       "    4 outputs             4+1       0.54$\\pm$0.07  0.85$\\pm$0.05\n",
       "    Baseline prevalence   1+1       0.50$\\pm$0.00  0.84$\\pm$0.04\n",
       "    Intermediate features 1024+1    0.50$\\pm$0.00  0.84$\\pm$0.04\n",
       "    18 outputs            18+1      0.47$\\pm$0.03  0.84$\\pm$0.03"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Americas, Asia, Europe\n"
     ]
    }
   ],
   "source": [
    "print(\", \".join(sorted(test_regions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Americas, Asia, Europe'"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\", \".join(sorted(results.test_region.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Survival\\\\N=53\n"
     ]
    }
   ],
   "source": [
    "print(target_str + \"\\\\\\\\N=\" + str(labels_mask.sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Survival\\\\0<offset<8 days\\\\True=46\\\\False=7\n"
     ]
    }
   ],
   "source": [
    "print(target_str + \"\\\\\\\\\" + \"0<offset<8 days\" + \n",
    "                   \"\\\\\\\\\" + \"True=\" + str(labels[labels_mask].value_counts()[True]) + \n",
    "                   \"\\\\\\\\\" + \"False=\" + str(labels[labels_mask].value_counts()[False]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(a.to_latex(escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}