XGboost_hold_karius_out.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hold-out validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.90 0.34.0\n"
     ]
    }
   ],
   "source": [
    "import xgboost\n",
    "import shap\n",
    "print(xgboost.__version__, shap.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Statements\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from pathlib import Path\n",
    "\n",
    "cwd = Path.cwd()\n",
    "datasets = cwd / 'datasets'\n",
    "results = cwd / 'results/pooled'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "Since we are using stratified kfold, a validation split is not necesssary."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>y</th>\n",
       "      <th>Bifidobacterium</th>\n",
       "      <th>Alloscardovia</th>\n",
       "      <th>Arthrobacter</th>\n",
       "      <th>Kocuria</th>\n",
       "      <th>Glutamicibacter</th>\n",
       "      <th>Citricoccus</th>\n",
       "      <th>Micrococcus</th>\n",
       "      <th>Pseudarthrobacter</th>\n",
       "      <th>...</th>\n",
       "      <th>Slackia</th>\n",
       "      <th>Mumia</th>\n",
       "      <th>Thermomonospora</th>\n",
       "      <th>Ilumatobacter</th>\n",
       "      <th>Marinibacterium</th>\n",
       "      <th>Maricaulis</th>\n",
       "      <th>Stella</th>\n",
       "      <th>Eoetvoesia</th>\n",
       "      <th>Bilophila</th>\n",
       "      <th>Paeniclostridium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>healthy</td>\n",
       "      <td>42891.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>septic</td>\n",
       "      <td>730.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>healthy</td>\n",
       "      <td>36074.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>healthy</td>\n",
       "      <td>44094.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>healthy</td>\n",
       "      <td>26958.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>karius</td>\n",
       "      <td>septic</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>karius</td>\n",
       "      <td>septic</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>karius</td>\n",
       "      <td>septic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>karius</td>\n",
       "      <td>septic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>karius</td>\n",
       "      <td>septic</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>438 rows × 687 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     dataset        y  Bifidobacterium  Alloscardovia  Arthrobacter  Kocuria  \\\n",
       "0    kapusta  healthy          42891.0            1.0          14.0      1.0   \n",
       "1    kapusta   septic            730.0            0.0          48.0      9.0   \n",
       "2    kapusta  healthy          36074.0            3.0           6.0     56.0   \n",
       "3    kapusta  healthy          44094.0            0.0          59.0     10.0   \n",
       "4    kapusta  healthy          26958.0            0.0           6.0      1.0   \n",
       "..       ...      ...              ...            ...           ...      ...   \n",
       "436   karius   septic              4.0            0.0           3.0      2.0   \n",
       "437   karius   septic              1.0            0.0           0.0      0.0   \n",
       "438   karius   septic              0.0            0.0           0.0      4.0   \n",
       "439   karius   septic              0.0            0.0           0.0      0.0   \n",
       "440   karius   septic              0.0            0.0           1.0      0.0   \n",
       "\n",
       "     Glutamicibacter  Citricoccus  Micrococcus  Pseudarthrobacter  ...  \\\n",
       "0                3.0          0.0         15.0                1.0  ...   \n",
       "1               10.0          1.0         20.0                5.0  ...   \n",
       "2                3.0          0.0         21.0                6.0  ...   \n",
       "3               21.0          3.0         66.0               40.0  ...   \n",
       "4                0.0          0.0         16.0                1.0  ...   \n",
       "..               ...          ...          ...                ...  ...   \n",
       "436              0.0          0.0          1.0                0.0  ...   \n",
       "437              0.0          0.0          1.0                0.0  ...   \n",
       "438              0.0          0.0          1.0                0.0  ...   \n",
       "439              0.0          0.0          0.0                0.0  ...   \n",
       "440              0.0          0.0          0.0                0.0  ...   \n",
       "\n",
       "     Slackia  Mumia  Thermomonospora  Ilumatobacter  Marinibacterium  \\\n",
       "0        0.0    0.0              0.0            7.0              1.0   \n",
       "1        0.0    0.0              0.0            0.0              0.0   \n",
       "2        0.0    0.0              0.0            0.0              0.0   \n",
       "3        0.0    0.0              0.0            0.0              0.0   \n",
       "4        0.0    0.0              0.0            0.0              0.0   \n",
       "..       ...    ...              ...            ...              ...   \n",
       "436     14.0    0.0              1.0            0.0              0.0   \n",
       "437      0.0    0.0              0.0            0.0              0.0   \n",
       "438      0.0    0.0              0.0            0.0              0.0   \n",
       "439      0.0    0.0              0.0            0.0              0.0   \n",
       "440      0.0    0.0              0.0            0.0              0.0   \n",
       "\n",
       "     Maricaulis  Stella  Eoetvoesia  Bilophila  Paeniclostridium  \n",
       "0           0.0     0.0         0.0        0.0               0.0  \n",
       "1           0.0     0.0         0.0        0.0               0.0  \n",
       "2           0.0     0.0         0.0        0.0               0.0  \n",
       "3           0.0     0.0         0.0        0.0               0.0  \n",
       "4           0.0     0.0         0.0        0.0               0.0  \n",
       "..          ...     ...         ...        ...               ...  \n",
       "436         0.0     0.0         0.0      109.0               0.0  \n",
       "437         0.0     0.0         0.0        0.0               0.0  \n",
       "438         0.0     0.0         0.0        0.0               0.0  \n",
       "439         0.0     0.0         0.0        0.0               0.0  \n",
       "440         0.0     0.0         0.0        0.0               0.0  \n",
       "\n",
       "[438 rows x 687 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>y</th>\n",
       "      <th>Bifidobacterium</th>\n",
       "      <th>Alloscardovia</th>\n",
       "      <th>Arthrobacter</th>\n",
       "      <th>Kocuria</th>\n",
       "      <th>Glutamicibacter</th>\n",
       "      <th>Citricoccus</th>\n",
       "      <th>Micrococcus</th>\n",
       "      <th>Pseudarthrobacter</th>\n",
       "      <th>...</th>\n",
       "      <th>Slackia</th>\n",
       "      <th>Mumia</th>\n",
       "      <th>Thermomonospora</th>\n",
       "      <th>Ilumatobacter</th>\n",
       "      <th>Marinibacterium</th>\n",
       "      <th>Maricaulis</th>\n",
       "      <th>Stella</th>\n",
       "      <th>Eoetvoesia</th>\n",
       "      <th>Bilophila</th>\n",
       "      <th>Paeniclostridium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>42891.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>1</td>\n",
       "      <td>730.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>36074.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>44094.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>26958.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>438 rows × 687 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     dataset  y  Bifidobacterium  Alloscardovia  Arthrobacter  Kocuria  \\\n",
       "0    kapusta  0          42891.0            1.0          14.0      1.0   \n",
       "1    kapusta  1            730.0            0.0          48.0      9.0   \n",
       "2    kapusta  0          36074.0            3.0           6.0     56.0   \n",
       "3    kapusta  0          44094.0            0.0          59.0     10.0   \n",
       "4    kapusta  0          26958.0            0.0           6.0      1.0   \n",
       "..       ... ..              ...            ...           ...      ...   \n",
       "436   karius  1              4.0            0.0           3.0      2.0   \n",
       "437   karius  1              1.0            0.0           0.0      0.0   \n",
       "438   karius  1              0.0            0.0           0.0      4.0   \n",
       "439   karius  1              0.0            0.0           0.0      0.0   \n",
       "440   karius  1              0.0            0.0           1.0      0.0   \n",
       "\n",
       "     Glutamicibacter  Citricoccus  Micrococcus  Pseudarthrobacter  ...  \\\n",
       "0                3.0          0.0         15.0                1.0  ...   \n",
       "1               10.0          1.0         20.0                5.0  ...   \n",
       "2                3.0          0.0         21.0                6.0  ...   \n",
       "3               21.0          3.0         66.0               40.0  ...   \n",
       "4                0.0          0.0         16.0                1.0  ...   \n",
       "..               ...          ...          ...                ...  ...   \n",
       "436              0.0          0.0          1.0                0.0  ...   \n",
       "437              0.0          0.0          1.0                0.0  ...   \n",
       "438              0.0          0.0          1.0                0.0  ...   \n",
       "439              0.0          0.0          0.0                0.0  ...   \n",
       "440              0.0          0.0          0.0                0.0  ...   \n",
       "\n",
       "     Slackia  Mumia  Thermomonospora  Ilumatobacter  Marinibacterium  \\\n",
       "0        0.0    0.0              0.0            7.0              1.0   \n",
       "1        0.0    0.0              0.0            0.0              0.0   \n",
       "2        0.0    0.0              0.0            0.0              0.0   \n",
       "3        0.0    0.0              0.0            0.0              0.0   \n",
       "4        0.0    0.0              0.0            0.0              0.0   \n",
       "..       ...    ...              ...            ...              ...   \n",
       "436     14.0    0.0              1.0            0.0              0.0   \n",
       "437      0.0    0.0              0.0            0.0              0.0   \n",
       "438      0.0    0.0              0.0            0.0              0.0   \n",
       "439      0.0    0.0              0.0            0.0              0.0   \n",
       "440      0.0    0.0              0.0            0.0              0.0   \n",
       "\n",
       "     Maricaulis  Stella  Eoetvoesia  Bilophila  Paeniclostridium  \n",
       "0           0.0     0.0         0.0        0.0               0.0  \n",
       "1           0.0     0.0         0.0        0.0               0.0  \n",
       "2           0.0     0.0         0.0        0.0               0.0  \n",
       "3           0.0     0.0         0.0        0.0               0.0  \n",
       "4           0.0     0.0         0.0        0.0               0.0  \n",
       "..          ...     ...         ...        ...               ...  \n",
       "436         0.0     0.0         0.0      109.0               0.0  \n",
       "437         0.0     0.0         0.0        0.0               0.0  \n",
       "438         0.0     0.0         0.0        0.0               0.0  \n",
       "439         0.0     0.0         0.0        0.0               0.0  \n",
       "440         0.0     0.0         0.0        0.0               0.0  \n",
       "\n",
       "[438 rows x 687 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>y</th>\n",
       "      <th>Bifidobacterium</th>\n",
       "      <th>Alloscardovia</th>\n",
       "      <th>Arthrobacter</th>\n",
       "      <th>Kocuria</th>\n",
       "      <th>Glutamicibacter</th>\n",
       "      <th>Citricoccus</th>\n",
       "      <th>Micrococcus</th>\n",
       "      <th>Pseudarthrobacter</th>\n",
       "      <th>...</th>\n",
       "      <th>Slackia</th>\n",
       "      <th>Mumia</th>\n",
       "      <th>Thermomonospora</th>\n",
       "      <th>Ilumatobacter</th>\n",
       "      <th>Marinibacterium</th>\n",
       "      <th>Maricaulis</th>\n",
       "      <th>Stella</th>\n",
       "      <th>Eoetvoesia</th>\n",
       "      <th>Bilophila</th>\n",
       "      <th>Paeniclostridium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>karius</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>karius</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>karius</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>karius</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>karius</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>karius</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>287 rows × 687 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset  y  Bifidobacterium  Alloscardovia  Arthrobacter  Kocuria  \\\n",
       "154  karius  0              1.0            0.0           2.0      1.0   \n",
       "155  karius  0              2.0            0.0           1.0      1.0   \n",
       "156  karius  0              0.0            0.0           2.0      3.0   \n",
       "157  karius  0              0.0            1.0           6.0      4.0   \n",
       "158  karius  0              6.0            0.0           4.0      8.0   \n",
       "..      ... ..              ...            ...           ...      ...   \n",
       "436  karius  1              4.0            0.0           3.0      2.0   \n",
       "437  karius  1              1.0            0.0           0.0      0.0   \n",
       "438  karius  1              0.0            0.0           0.0      4.0   \n",
       "439  karius  1              0.0            0.0           0.0      0.0   \n",
       "440  karius  1              0.0            0.0           1.0      0.0   \n",
       "\n",
       "     Glutamicibacter  Citricoccus  Micrococcus  Pseudarthrobacter  ...  \\\n",
       "154              0.0          0.0          2.0                0.0  ...   \n",
       "155              0.0          0.0          1.0                0.0  ...   \n",
       "156              0.0          0.0          0.0                0.0  ...   \n",
       "157              0.0          0.0          0.0                6.0  ...   \n",
       "158              1.0          0.0          2.0                0.0  ...   \n",
       "..               ...          ...          ...                ...  ...   \n",
       "436              0.0          0.0          1.0                0.0  ...   \n",
       "437              0.0          0.0          1.0                0.0  ...   \n",
       "438              0.0          0.0          1.0                0.0  ...   \n",
       "439              0.0          0.0          0.0                0.0  ...   \n",
       "440              0.0          0.0          0.0                0.0  ...   \n",
       "\n",
       "     Slackia  Mumia  Thermomonospora  Ilumatobacter  Marinibacterium  \\\n",
       "154      0.0    0.0              0.0            0.0              0.0   \n",
       "155      0.0    0.0              0.0            0.0              0.0   \n",
       "156      0.0    0.0              0.0            0.0              0.0   \n",
       "157      0.0    0.0              0.0            0.0              0.0   \n",
       "158      0.0    0.0              0.0            0.0              0.0   \n",
       "..       ...    ...              ...            ...              ...   \n",
       "436     14.0    0.0              1.0            0.0              0.0   \n",
       "437      0.0    0.0              0.0            0.0              0.0   \n",
       "438      0.0    0.0              0.0            0.0              0.0   \n",
       "439      0.0    0.0              0.0            0.0              0.0   \n",
       "440      0.0    0.0              0.0            0.0              0.0   \n",
       "\n",
       "     Maricaulis  Stella  Eoetvoesia  Bilophila  Paeniclostridium  \n",
       "154         0.0     0.0         0.0        0.0               0.0  \n",
       "155         0.0     0.0         0.0        0.0               0.0  \n",
       "156         0.0     0.0         0.0        0.0               0.0  \n",
       "157         0.0     0.0         0.0        0.0               0.0  \n",
       "158         0.0     0.0         0.0        0.0               0.0  \n",
       "..          ...     ...         ...        ...               ...  \n",
       "436         0.0     0.0         0.0      109.0               0.0  \n",
       "437         0.0     0.0         0.0        0.0               0.0  \n",
       "438         0.0     0.0         0.0        0.0               0.0  \n",
       "439         0.0     0.0         0.0        0.0               0.0  \n",
       "440         0.0     0.0         0.0        0.0               0.0  \n",
       "\n",
       "[287 rows x 687 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>y</th>\n",
       "      <th>Bifidobacterium</th>\n",
       "      <th>Alloscardovia</th>\n",
       "      <th>Arthrobacter</th>\n",
       "      <th>Kocuria</th>\n",
       "      <th>Glutamicibacter</th>\n",
       "      <th>Citricoccus</th>\n",
       "      <th>Micrococcus</th>\n",
       "      <th>Pseudarthrobacter</th>\n",
       "      <th>...</th>\n",
       "      <th>Slackia</th>\n",
       "      <th>Mumia</th>\n",
       "      <th>Thermomonospora</th>\n",
       "      <th>Ilumatobacter</th>\n",
       "      <th>Marinibacterium</th>\n",
       "      <th>Maricaulis</th>\n",
       "      <th>Stella</th>\n",
       "      <th>Eoetvoesia</th>\n",
       "      <th>Bilophila</th>\n",
       "      <th>Paeniclostridium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>42891.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>1</td>\n",
       "      <td>730.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>36074.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>44094.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>kapusta</td>\n",
       "      <td>0</td>\n",
       "      <td>26958.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>grumaz</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>grumaz</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>grumaz</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>grumaz</td>\n",
       "      <td>1</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>grumaz</td>\n",
       "      <td>1</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151 rows × 687 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     dataset  y  Bifidobacterium  Alloscardovia  Arthrobacter  Kocuria  \\\n",
       "0    kapusta  0          42891.0            1.0          14.0      1.0   \n",
       "1    kapusta  1            730.0            0.0          48.0      9.0   \n",
       "2    kapusta  0          36074.0            3.0           6.0     56.0   \n",
       "3    kapusta  0          44094.0            0.0          59.0     10.0   \n",
       "4    kapusta  0          26958.0            0.0           6.0      1.0   \n",
       "..       ... ..              ...            ...           ...      ...   \n",
       "149   grumaz  1              0.0            0.0           0.0      0.0   \n",
       "150   grumaz  1              0.0            0.0           0.0      0.0   \n",
       "151   grumaz  1              0.0            0.0           0.0      0.0   \n",
       "152   grumaz  1             23.0            0.0           0.0      9.0   \n",
       "153   grumaz  1             25.0            0.0           2.0     13.0   \n",
       "\n",
       "     Glutamicibacter  Citricoccus  Micrococcus  Pseudarthrobacter  ...  \\\n",
       "0                3.0          0.0         15.0                1.0  ...   \n",
       "1               10.0          1.0         20.0                5.0  ...   \n",
       "2                3.0          0.0         21.0                6.0  ...   \n",
       "3               21.0          3.0         66.0               40.0  ...   \n",
       "4                0.0          0.0         16.0                1.0  ...   \n",
       "..               ...          ...          ...                ...  ...   \n",
       "149              0.0          0.0          0.0                0.0  ...   \n",
       "150              0.0          0.0          0.0                0.0  ...   \n",
       "151              0.0          0.0          6.0                0.0  ...   \n",
       "152              0.0          0.0          7.0                0.0  ...   \n",
       "153              0.0          0.0         25.0                0.0  ...   \n",
       "\n",
       "     Slackia  Mumia  Thermomonospora  Ilumatobacter  Marinibacterium  \\\n",
       "0        0.0    0.0              0.0            7.0              1.0   \n",
       "1        0.0    0.0              0.0            0.0              0.0   \n",
       "2        0.0    0.0              0.0            0.0              0.0   \n",
       "3        0.0    0.0              0.0            0.0              0.0   \n",
       "4        0.0    0.0              0.0            0.0              0.0   \n",
       "..       ...    ...              ...            ...              ...   \n",
       "149      0.0    0.0              0.0            0.0              0.0   \n",
       "150      0.0    0.0              0.0            0.0              0.0   \n",
       "151      0.0    0.0              0.0            0.0              0.0   \n",
       "152      0.0    0.0              0.0            0.0              0.0   \n",
       "153      0.0    0.0              0.0            0.0              0.0   \n",
       "\n",
       "     Maricaulis  Stella  Eoetvoesia  Bilophila  Paeniclostridium  \n",
       "0           0.0     0.0         0.0        0.0               0.0  \n",
       "1           0.0     0.0         0.0        0.0               0.0  \n",
       "2           0.0     0.0         0.0        0.0               0.0  \n",
       "3           0.0     0.0         0.0        0.0               0.0  \n",
       "4           0.0     0.0         0.0        0.0               0.0  \n",
       "..          ...     ...         ...        ...               ...  \n",
       "149         0.0     0.0         0.0        0.0               0.0  \n",
       "150         0.0     0.0         0.0        0.0               0.0  \n",
       "151         0.0     0.0         0.0        0.0               0.0  \n",
       "152         0.0     0.0         0.0        0.0               0.0  \n",
       "153         0.0     0.0         0.0        0.0               0.0  \n",
       "\n",
       "[151 rows x 687 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "raw_df = pd.read_csv(datasets / 'kapusta_grumaz_karius_genus_raw.csv')\n",
    "\n",
    "# Remove NTCs\n",
    "raw_df = raw_df.loc[raw_df.y != 'ntc', :]\n",
    "display(raw_df)\n",
    "\n",
    "# Binary encode y\n",
    "raw_df.loc[raw_df.y == 'septic', 'y'] = 1\n",
    "raw_df.loc[raw_df.y == 'healthy', 'y'] = 0\n",
    "raw_df = raw_df.astype({'y': 'int'})\n",
    "\n",
    "# Get hold out set\n",
    "hold_out = 'karius'\n",
    "holdout_df = raw_df.loc[raw_df.dataset == hold_out, :]\n",
    "other_df = raw_df.loc[raw_df.dataset != hold_out, :]\n",
    "\n",
    "holdout_X = holdout_df.drop(['y', 'dataset'], axis=1).copy()\n",
    "holdout_y = holdout_df.y.copy()\n",
    "\n",
    "other_X = other_df.drop(['y', 'dataset'], axis=1).copy()\n",
    "other_y = other_df.y.copy()\n",
    "\n",
    "# Relative abundance\n",
    "other_X_RA = other_X.apply(func=lambda x: x / x.sum(), axis=1)\n",
    "holdout_X_RA = holdout_X.apply(func=lambda x: x / x.sum(), axis=1)\n",
    "\n",
    "display(raw_df)\n",
    "display(holdout_df)\n",
    "display(other_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute train test split size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_splits = 10\n",
    "\n",
    "pos = len(other_y[other_y == 1])\n",
    "neg = len(other_y[other_y == 0])\n",
    "test_pos = len(holdout_y[holdout_y == 1])\n",
    "test_neg = len(holdout_y[holdout_y == 0])\n",
    "split_sizes = pd.DataFrame({'Septic': [pos - int(pos / n_splits), int(pos / n_splits), test_pos], \n",
    "                           'Healthy': [neg - int(neg / n_splits), int(neg / n_splits), test_neg]}, \n",
    "                           index=['Train fold', 'Validation fold', 'Test fold'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nested CV for hyperparameter optimisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold, cross_validate\n",
    "from sklearn.metrics import make_scorer, precision_score, recall_score, f1_score, average_precision_score, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimise_evaluate(X, y):\n",
    "    np.random.seed(66)\n",
    "    ratio = sum(y == 0) / sum(y == 1)\n",
    "    \n",
    "    # Hyperparemeter Optimisation using grid search (F1)\n",
    "    model = XGBClassifier()\n",
    "    n_estimators = range(100, 300, 10)\n",
    "    max_depth = range(1, 5, 1)\n",
    "    gamma = np.linspace(0.1, 3, 10)\n",
    "    subsample = [0.6, 0.7, 0.8, 0.9, 1.0]\n",
    "    colsample_bytree = np.linspace(0.1, 1, 20)\n",
    "    \n",
    "    param_grid = dict(max_depth=max_depth, \n",
    "                      n_estimators=n_estimators, \n",
    "                      colsample_bytree=colsample_bytree,\n",
    "                      gamma=gamma,\n",
    "                      subsample=subsample,\n",
    "                      scale_pos_weight=[ratio])\n",
    "    \n",
    "    inner_cv = StratifiedKFold(n_splits=n_splits, shuffle=True)\n",
    "    outer_cv = StratifiedKFold(n_splits=n_splits, shuffle=True)\n",
    "\n",
    "    # Inner CV\n",
    "    model = RandomizedSearchCV(model, \n",
    "                               param_grid, \n",
    "                               scoring='roc_auc',\n",
    "                               n_iter=1000,\n",
    "                               n_jobs=70, \n",
    "                               cv=inner_cv, \n",
    "                               verbose=0)\n",
    "\n",
    "    model.fit(X, y)\n",
    "    best_params = model.best_params_\n",
    "    print(best_params)\n",
    "\n",
    "    # Custom metrics\n",
    "    precision = make_scorer(precision_score, average='binary')\n",
    "    recall = make_scorer(recall_score, average='binary')\n",
    "    f1 = make_scorer(f1_score, average='binary')\n",
    "    auprc = make_scorer(average_precision_score, average=None)\n",
    "    \n",
    "    scoring = {'precision': precision, \n",
    "               'recall': recall, \n",
    "               'AUROC': 'roc_auc',\n",
    "               'AUPRC': auprc,\n",
    "               'F1': f1}\n",
    "\n",
    "    # Outer CV\n",
    "    outer_results = cross_validate(model, X=X, y=y, cv=outer_cv, scoring=scoring)\n",
    "    outer_results = pd.DataFrame(outer_results).mean()[['test_precision', 'test_recall', 'test_F1', 'test_AUROC', 'test_AUPRC']]\n",
    "\n",
    "    return outer_results, best_params\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Optimise and evaluate clean models trained on Grumaz + Kapusta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'subsample': 0.7, 'scale_pos_weight': 0.336283185840708, 'n_estimators': 200, 'max_depth': 3, 'gamma': 0.42222222222222217, 'colsample_bytree': 0.1}\n",
      "{'subsample': 0.7, 'scale_pos_weight': 0.336283185840708, 'n_estimators': 270, 'max_depth': 4, 'gamma': 2.033333333333333, 'colsample_bytree': 0.1473684210526316}\n"
     ]
    }
   ],
   "source": [
    "raw_results, raw_params = optimise_evaluate(other_X, other_y)\n",
    "# raw_params = {'subsample': 0.5263157894736842, 'scale_pos_weight': 1.4273504273504274, 'n_estimators': 96, 'max_depth': 2, 'gamma': 1.8421052631578947, 'colsample_bytree': 0.19473684210526315}\n",
    "\n",
    "RA_results, RA_params = optimise_evaluate(other_X_RA, other_y)\n",
    "# RA_params = {'subsample': 0.4789473684210527, 'scale_pos_weight': 1.4273504273504274, 'n_estimators': 255, 'max_depth': 1, 'gamma': 0.7894736842105263, 'colsample_bytree': 0.4}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Decontamination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fit dirty model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(colsample_bytree=0.1, gamma=0.42222222222222217, n_estimators=200,\n",
       "              scale_pos_weight=0.336283185840708, subsample=0.7)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_model = XGBClassifier(**raw_params)\n",
    "raw_model.fit(other_X, other_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove non-human associated pathogens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieve known human pathogens\n",
    "meta = pd.read_csv(datasets / 'pathogen_list.csv', encoding= 'unicode_escape')\n",
    "meta = meta['Genus'].unique()\n",
    "\n",
    "# Remove non-human pathogens\n",
    "genera_new = other_X.columns\n",
    "genera_new = list(set(genera_new).intersection(set(meta)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove contaminants based on SHAP values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from scipy.stats import spearmanr\n",
    "import shap\n",
    "\n",
    "\n",
    "def decontam(X_train, y_train, params):\n",
    "    X_train = X_train.copy()\n",
    "    y_train = y_train.copy()\n",
    "    params = params.copy()\n",
    "    \n",
    "    model = XGBClassifier(**params)\n",
    "    model.fit(X=X_train, y=y_train)\n",
    "\n",
    "    explainer = shap.TreeExplainer(model, feature_pertubation='interventional', model_output='probability', data=X_train)\n",
    "    shap_val = explainer.shap_values(X_train)\n",
    "\n",
    "    to_retain = np.array([True] * X_train.shape[1])\n",
    "    corrs = np.zeros(X_train.shape[1])\n",
    "    \n",
    "    for i in range(X_train.shape[1]):\n",
    "        if sum(X_train.iloc[:, i] != 0) >= X_train.shape[0] * 0.1:\n",
    "            rho = spearmanr(X_train.iloc[:, i], shap_val[:, i])[0]\n",
    "            p = spearmanr(X_train.iloc[:, i], shap_val[:, i])[1]\n",
    "            print(f'rho={rho}, p={p}, genus={X_train.columns[i]}')\n",
    "            \n",
    "            if rho < 0:\n",
    "                to_retain[i] = False\n",
    "                \n",
    "            elif not shap_val[:, i].any(0):\n",
    "                to_retain[i] = False\n",
    "        else:\n",
    "            to_retain[i] = False\n",
    "\n",
    "    to_retain = X_train.columns[to_retain]\n",
    "    print(to_retain.shape, to_retain)\n",
    "    \n",
    "    return to_retain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rho=-0.277021855972936, p=0.0005743255898167671, genus=Bifidobacterium\n",
      "rho=0.7110258291812392, p=1.4844824830087602e-24, genus=Delftia\n",
      "rho=-0.7770133926838283, p=9.355095088070242e-32, genus=Staphylococcus\n",
      "rho=nan, p=nan, genus=Leifsonia\n",
      "rho=nan, p=nan, genus=Psychrobacter\n",
      "rho=-0.7592988916065453, p=1.349080202138056e-29, genus=Lactococcus\n",
      "rho=nan, p=nan, genus=Halomonas\n",
      "rho=nan, p=nan, genus=Shewanella\n",
      "rho=nan, p=nan, genus=Moraxella\n",
      "rho=nan, p=nan, genus=Porphyromonas\n",
      "rho=0.4003672246867435, p=3.513086119741646e-07, genus=Sphingomonas\n",
      "rho=-0.296865323868207, p=0.00021439027361914886, genus=Flavobacterium\n",
      "rho=-0.16961219384971984, p=0.03734064789915684, genus=Afipia\n",
      "rho=nan, p=nan, genus=Carnobacterium\n",
      "rho=0.7896816336966194, p=2.002599117065124e-33, genus=Stenotrophomonas\n",
      "rho=-0.7481746876432195, p=2.470899814279914e-28, genus=Lachnoclostridium\n",
      "rho=-0.5982592238090015, p=4.993448113056925e-16, genus=Mycoplasma\n",
      "rho=nan, p=nan, genus=Gemella\n",
      "rho=-0.6377207028099147, p=1.3046595778340145e-18, genus=Acinetobacter\n",
      "rho=0.6283377860846313, p=5.7961990205886754e-18, genus=Bacillus\n",
      "rho=nan, p=nan, genus=Actinomadura\n",
      "rho=-0.8112152803289489, p=1.521598831492488e-36, genus=Chryseobacterium\n",
      "rho=nan, p=nan, genus=Cupriavidus\n",
      "rho=nan, p=nan, genus=Aerococcus\n",
      "rho=0.8519697002773783, p=1.030289648283762e-43, genus=Rhodococcus\n",
      "rho=nan, p=nan, genus=Peptoniphilus\n",
      "rho=nan, p=nan, genus=Lysinibacillus\n",
      "rho=nan, p=nan, genus=Bordetella\n",
      "rho=0.7674563217666411, p=1.4442598994234977e-30, genus=Dermabacter\n",
      "rho=-0.2685902520001403, p=0.0008543569621892071, genus=Arthrobacter\n",
      "rho=nan, p=nan, genus=Listeria\n",
      "rho=nan, p=nan, genus=Pantoea\n",
      "rho=nan, p=nan, genus=Azospirillum\n",
      "rho=0.6871997798185189, p=1.9755035565234853e-22, genus=Morganella\n",
      "rho=-0.7644671784910061, p=3.3096434728997435e-30, genus=Massilia\n",
      "rho=-0.253788916943242, p=0.0016646177736606798, genus=Cronobacter\n",
      "rho=nan, p=nan, genus=Brucella\n",
      "rho=nan, p=nan, genus=Dietzia\n",
      "rho=nan, p=nan, genus=Gordonia\n",
      "rho=-0.25548902896484443, p=0.0015448231241109772, genus=Ochrobactrum\n",
      "rho=nan, p=nan, genus=Propionibacterium\n",
      "rho=nan, p=nan, genus=Achromobacter\n",
      "rho=nan, p=nan, genus=Dysgonomonas\n",
      "rho=nan, p=nan, genus=Photobacterium\n",
      "rho=nan, p=nan, genus=Providencia\n",
      "rho=0.636054930492254, p=1.7064748917200373e-18, genus=Microbacterium\n",
      "rho=0.4088365171280704, p=1.8710299332088994e-07, genus=Bosea\n",
      "rho=-0.6633383117638615, p=1.691532541949996e-20, genus=Dyella\n",
      "rho=nan, p=nan, genus=Raoultella\n",
      "rho=-0.6296977587025402, p=4.6842107373601016e-18, genus=Micrococcus\n",
      "rho=-0.7943798437438366, p=4.4971681805214804e-34, genus=Streptococcus\n",
      "rho=0.6435238536155188, p=5.054261273376406e-19, genus=Bartonella\n",
      "rho=0.7469034268453045, p=3.411635363933372e-28, genus=Comamonas\n",
      "rho=0.7091248678428336, p=2.232982714081086e-24, genus=Pseudoclavibacter\n",
      "rho=0.660126347680094, p=2.98546119552842e-20, genus=Cellulosimicrobium\n",
      "rho=nan, p=nan, genus=Weissella\n",
      "rho=0.590180215498889, p=1.5287372767006632e-15, genus=Rothia\n",
      "rho=nan, p=nan, genus=Proteus\n",
      "rho=nan, p=nan, genus=Janibacter\n",
      "rho=nan, p=nan, genus=Brevibacterium\n",
      "rho=nan, p=nan, genus=Campylobacter\n",
      "rho=0.7187651001513332, p=2.7206190767678738e-25, genus=Serratia\n",
      "rho=-0.02938709059611943, p=0.7202002395885738, genus=Yersinia\n",
      "rho=nan, p=nan, genus=Granulicatella\n",
      "rho=nan, p=nan, genus=Luteibacter\n",
      "rho=0.7126817076971801, p=1.0374186984306752e-24, genus=Oerskovia\n",
      "rho=nan, p=nan, genus=Haemophilus\n",
      "rho=nan, p=nan, genus=Bacteroides\n",
      "rho=-0.8723814546042615, p=3.60552402130785e-48, genus=Paracoccus\n",
      "rho=0.6617410379263124, p=2.2457196784417648e-20, genus=Aeromonas\n",
      "rho=0.5151485945395603, p=1.3086231275606088e-11, genus=Enterobacter\n",
      "rho=nan, p=nan, genus=Erwinia\n",
      "rho=nan, p=nan, genus=Neisseria\n",
      "rho=nan, p=nan, genus=Sphingobacterium\n",
      "rho=nan, p=nan, genus=Pseudonocardia\n",
      "rho=nan, p=nan, genus=Collinsella\n",
      "rho=nan, p=nan, genus=Blautia\n",
      "rho=nan, p=nan, genus=Arsenicicoccus\n",
      "rho=-0.5308988566188995, p=2.3465413307705818e-12, genus=Chitinophaga\n",
      "rho=-0.8458311710154363, p=1.6720328784821492e-42, genus=Lactobacillus\n",
      "rho=nan, p=nan, genus=Alistipes\n",
      "rho=nan, p=nan, genus=Kytococcus\n",
      "rho=nan, p=nan, genus=Helicobacter\n",
      "rho=-0.7671468803953219, p=1.5746245041318123e-30, genus=Anaerococcus\n",
      "rho=0.7462041522245904, p=4.070758519476137e-28, genus=Citrobacter\n",
      "rho=0.6962335387621638, p=3.271558783123744e-23, genus=Klebsiella\n",
      "rho=-0.7612012756924748, p=8.076241781303949e-30, genus=Actinomyces\n",
      "rho=0.4694649640334048, p=1.1965536326888372e-09, genus=Cellulomonas\n",
      "rho=nan, p=nan, genus=Nocardiopsis\n",
      "rho=nan, p=nan, genus=Vagococcus\n",
      "rho=-0.6509747443415187, p=1.4511890132994526e-19, genus=Veillonella\n",
      "rho=nan, p=nan, genus=Finegoldia\n",
      "rho=nan, p=nan, genus=Mycobacterium\n",
      "rho=nan, p=nan, genus=Rickettsia\n",
      "rho=nan, p=nan, genus=Sutterella\n",
      "rho=nan, p=nan, genus=Alcaligenes\n",
      "rho=nan, p=nan, genus=Pandoraea\n",
      "rho=nan, p=nan, genus=Amycolatopsis\n",
      "rho=-0.6577564619473104, p=4.5199109915083765e-20, genus=Facklamia\n",
      "rho=-0.7202950155191093, p=1.9322499176983279e-25, genus=Desulfovibrio\n",
      "rho=nan, p=nan, genus=Leptotrichia\n",
      "rho=nan, p=nan, genus=Dermacoccus\n",
      "rho=-0.6282419763450026, p=5.883601121814517e-18, genus=Streptomyces\n",
      "rho=-0.3254955524524073, p=4.539748379308031e-05, genus=Brevundimonas\n",
      "rho=nan, p=nan, genus=Nocardia\n",
      "rho=nan, p=nan, genus=Aureimonas\n",
      "rho=nan, p=nan, genus=Treponema\n",
      "rho=nan, p=nan, genus=Saccharomonospora\n",
      "rho=0.7183195776881632, p=3.004398098380576e-25, genus=Enterococcus\n",
      "rho=-0.5011861399144975, p=5.5875286819951446e-11, genus=Clostridioides\n",
      "rho=0.5794106654957681, p=6.477928853377124e-15, genus=Pseudomonas\n",
      "rho=nan, p=nan, genus=Prevotella\n",
      "rho=0.5359704882854112, p=1.3236730092318285e-12, genus=Brachybacterium\n",
      "rho=nan, p=nan, genus=Roseomonas\n",
      "rho=0.7246737248865632, p=7.165236771481646e-26, genus=Salmonella\n",
      "rho=-0.8042790744874212, p=1.6953430160882268e-35, genus=Corynebacterium\n",
      "rho=nan, p=nan, genus=Exiguobacterium\n",
      "rho=nan, p=nan, genus=Vibrio\n",
      "rho=nan, p=nan, genus=Fusobacterium\n",
      "rho=0.7599780663834814, p=1.1238877223167303e-29, genus=Curtobacterium\n",
      "rho=nan, p=nan, genus=Ralstonia\n",
      "rho=0.6173720984450273, p=3.109707014208e-17, genus=Kocuria\n",
      "rho=nan, p=nan, genus=Legionella\n",
      "rho=-0.7959074860622659, p=2.7439683555624263e-34, genus=Paenibacillus\n",
      "rho=-0.8006262889742016, p=5.806404103469061e-35, genus=Leuconostoc\n",
      "rho=-0.5733313410172473, p=1.430169755729748e-14, genus=Brevibacillus\n",
      "rho=0.4180321990213671, p=9.253960422362766e-08, genus=Bradyrhizobium\n",
      "(28,) Index(['Delftia', 'Sphingomonas', 'Stenotrophomonas', 'Bacillus',\n",
      "       'Rhodococcus', 'Dermabacter', 'Morganella', 'Microbacterium', 'Bosea',\n",
      "       'Bartonella', 'Comamonas', 'Pseudoclavibacter', 'Cellulosimicrobium',\n",
      "       'Rothia', 'Serratia', 'Oerskovia', 'Aeromonas', 'Enterobacter',\n",
      "       'Citrobacter', 'Klebsiella', 'Cellulomonas', 'Enterococcus',\n",
      "       'Pseudomonas', 'Brachybacterium', 'Salmonella', 'Curtobacterium',\n",
      "       'Kocuria', 'Bradyrhizobium'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "An input array is constant; the correlation coefficent is not defined.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rho=0.1219073249338576, p=0.13592439262414596, genus=Delftia\n",
      "rho=0.002601592732436892, p=0.9747086612523781, genus=Sphingomonas\n",
      "rho=0.7600535738997743, p=1.1012573409801312e-29, genus=Stenotrophomonas\n",
      "rho=-0.7352155238016681, p=6.0689660159077266e-27, genus=Bacillus\n",
      "rho=0.7507942532481674, p=1.263250216934176e-28, genus=Rhodococcus\n",
      "rho=0.7538201578335659, p=5.759553820064766e-29, genus=Dermabacter\n",
      "rho=0.6864803072911557, p=2.2732885930698107e-22, genus=Morganella\n",
      "rho=-0.3048055867558833, p=0.0001416054543844417, genus=Microbacterium\n",
      "rho=-0.6481319103723334, p=2.345701998043673e-19, genus=Bosea\n",
      "rho=0.173474418671955, p=0.03315897050312701, genus=Bartonella\n",
      "rho=0.7024624809002671, p=9.103898925910515e-24, genus=Comamonas\n",
      "rho=0.6781525404564123, p=1.1216275477470191e-21, genus=Pseudoclavibacter\n",
      "rho=0.6982343237363573, p=2.1770546274956248e-23, genus=Cellulosimicrobium\n",
      "rho=-0.5142230765832161, p=1.4437509416635619e-11, genus=Rothia\n",
      "rho=0.5824140713379595, p=4.353792138589947e-15, genus=Serratia\n",
      "rho=0.8009659659853514, p=5.183928336484325e-35, genus=Oerskovia\n",
      "rho=0.5108054435473273, p=2.0700394718246117e-11, genus=Aeromonas\n",
      "rho=-0.6022602097516024, p=2.835688923015103e-16, genus=Enterobacter\n",
      "rho=0.8629570977194738, p=5.053285672196494e-46, genus=Citrobacter\n",
      "rho=0.4531412633419735, p=5.1465348086652905e-09, genus=Klebsiella\n",
      "rho=0.279538729221147, p=0.0005088654055874355, genus=Cellulomonas\n",
      "rho=-0.7549376423517249, p=4.297006731740729e-29, genus=Enterococcus\n",
      "rho=0.37614419940047017, p=1.938705934120565e-06, genus=Pseudomonas\n",
      "rho=-0.7104606944193963, p=1.6766281674784483e-24, genus=Brachybacterium\n",
      "rho=0.8410235599737367, p=1.3640817351729879e-41, genus=Salmonella\n",
      "rho=0.8399918691689727, p=2.120893389151544e-41, genus=Curtobacterium\n",
      "rho=-0.47071046080425094, p=1.067109716776606e-09, genus=Kocuria\n",
      "rho=-0.3709967902192292, p=2.739254489675715e-06, genus=Bradyrhizobium\n",
      "(19,) Index(['Delftia', 'Sphingomonas', 'Stenotrophomonas', 'Rhodococcus',\n",
      "       'Dermabacter', 'Morganella', 'Bartonella', 'Comamonas',\n",
      "       'Pseudoclavibacter', 'Cellulosimicrobium', 'Serratia', 'Oerskovia',\n",
      "       'Aeromonas', 'Citrobacter', 'Klebsiella', 'Cellulomonas', 'Pseudomonas',\n",
      "       'Salmonella', 'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=-0.11845674380927786, p=0.1474408245036441, genus=Delftia\n",
      "rho=-0.5080918535351767, p=2.7478729405118198e-11, genus=Sphingomonas\n",
      "rho=0.7410890434762318, p=1.456425017925062e-27, genus=Stenotrophomonas\n",
      "rho=0.7183706293024456, p=2.9704627889205365e-25, genus=Rhodococcus\n",
      "rho=0.7548654611431933, p=4.379296117933775e-29, genus=Dermabacter\n",
      "rho=0.6850190082496037, p=3.0196888649859064e-22, genus=Morganella\n",
      "rho=0.11992637835188191, p=0.1424487229623762, genus=Bartonella\n",
      "rho=-0.38313593615272373, p=1.2007323944484019e-06, genus=Comamonas\n",
      "rho=0.5356909917882664, p=1.3664438586584181e-12, genus=Pseudoclavibacter\n",
      "rho=0.6142304240283027, p=4.971953780779433e-17, genus=Cellulosimicrobium\n",
      "rho=0.5936588581411731, p=9.47905560345869e-16, genus=Serratia\n",
      "rho=0.8044242663645049, p=1.6135167623839457e-35, genus=Oerskovia\n",
      "rho=0.37214476339670305, p=2.537320496908193e-06, genus=Aeromonas\n",
      "rho=0.7173734671898992, p=3.706704095496879e-25, genus=Citrobacter\n",
      "rho=-0.2914084810166235, p=0.00028315259855354175, genus=Klebsiella\n",
      "rho=0.45897530871922776, p=3.0823889926662087e-09, genus=Cellulomonas\n",
      "rho=0.31339044805773664, p=8.922167912034297e-05, genus=Pseudomonas\n",
      "rho=nan, p=nan, genus=Salmonella\n",
      "rho=0.6614517354248597, p=2.3635616526066053e-20, genus=Curtobacterium\n",
      "(14,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Bartonella', 'Pseudoclavibacter', 'Cellulosimicrobium', 'Serratia',\n",
      "       'Oerskovia', 'Aeromonas', 'Citrobacter', 'Cellulomonas', 'Pseudomonas',\n",
      "       'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.6712762695433013, p=4.0298195248720455e-21, genus=Stenotrophomonas\n",
      "rho=0.7388615268049897, p=2.513714797159661e-27, genus=Rhodococcus\n",
      "rho=0.7546118797892232, p=4.6808395125897493e-29, genus=Dermabacter\n",
      "rho=0.6854858627021054, p=2.758331441061322e-22, genus=Morganella\n",
      "rho=-0.5986886647037943, p=4.700974210330125e-16, genus=Bartonella\n",
      "rho=0.6420806973216071, p=6.410635081655121e-19, genus=Pseudoclavibacter\n",
      "rho=0.6230209978879321, p=1.3196834232466974e-17, genus=Cellulosimicrobium\n",
      "rho=0.6160975234922345, p=3.7642506013007694e-17, genus=Serratia\n",
      "rho=0.7646675030352786, p=3.1319365677488168e-30, genus=Oerskovia\n",
      "rho=-0.16493594462556324, p=0.04299000660758382, genus=Aeromonas\n",
      "rho=0.7410843199018083, p=1.4581202218345952e-27, genus=Citrobacter\n",
      "rho=-0.2930791689634557, p=0.00026018744198427865, genus=Cellulomonas\n",
      "rho=0.3640184648623388, p=4.335837636109578e-06, genus=Pseudomonas\n",
      "rho=0.44435812293424964, p=1.0936770429604344e-08, genus=Curtobacterium\n",
      "(11,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Pseudoclavibacter', 'Cellulosimicrobium', 'Serratia', 'Oerskovia',\n",
      "       'Citrobacter', 'Pseudomonas', 'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.7402769766551759, p=1.7781988337477324e-27, genus=Stenotrophomonas\n",
      "rho=0.6800603371370998, p=7.817444409091254e-22, genus=Rhodococcus\n",
      "rho=0.7839449598710362, p=1.1789978846094428e-32, genus=Dermabacter\n",
      "rho=0.6850490867566479, p=3.0021430035731513e-22, genus=Morganella\n",
      "rho=0.21590986393692932, p=0.007754452719014847, genus=Pseudoclavibacter\n",
      "rho=0.6189351289418055, p=2.4573671830193603e-17, genus=Cellulosimicrobium\n",
      "rho=-0.4985212019965909, p=7.316991120833033e-11, genus=Serratia\n",
      "rho=0.7591247533614879, p=1.413616376441441e-29, genus=Oerskovia\n",
      "rho=0.8109653528544746, p=1.6625395856750833e-36, genus=Citrobacter\n",
      "rho=0.02240813606122219, p=0.7847748725081657, genus=Pseudomonas\n",
      "rho=0.3867310672157943, p=9.344795017848195e-07, genus=Curtobacterium\n",
      "(10,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Pseudoclavibacter', 'Cellulosimicrobium', 'Oerskovia', 'Citrobacter',\n",
      "       'Pseudomonas', 'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.697944532375635, p=2.3098268376924372e-23, genus=Stenotrophomonas\n",
      "rho=0.6701258071581861, p=4.974702669193334e-21, genus=Rhodococcus\n",
      "rho=0.7581276312908529, p=1.845882384481379e-29, genus=Dermabacter\n",
      "rho=0.6852443534989004, p=2.890640707462507e-22, genus=Morganella\n",
      "rho=0.4191048448578865, p=8.512628720701529e-08, genus=Pseudoclavibacter\n",
      "rho=0.6134607502502472, p=5.573292912520985e-17, genus=Cellulosimicrobium\n",
      "rho=0.8026663611386444, p=2.9287810639326023e-35, genus=Oerskovia\n",
      "rho=0.7006614472532091, p=1.3222855151297943e-23, genus=Citrobacter\n",
      "rho=-0.03448651540225867, p=0.6742071751850316, genus=Pseudomonas\n",
      "rho=0.3762677545579971, p=1.9225464246448937e-06, genus=Curtobacterium\n",
      "(9,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Pseudoclavibacter', 'Cellulosimicrobium', 'Oerskovia', 'Citrobacter',\n",
      "       'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.7307352292629213, p=1.7576827391474535e-26, genus=Stenotrophomonas\n",
      "rho=0.673049612812364, p=2.9071630050840914e-21, genus=Rhodococcus\n",
      "rho=0.7867381301440216, p=5.007335523044691e-33, genus=Dermabacter\n",
      "rho=0.6884046444904823, p=1.5601528524915246e-22, genus=Morganella\n",
      "rho=-0.7523039047118985, p=8.549210783147952e-29, genus=Pseudoclavibacter\n",
      "rho=0.6209878825482515, p=1.800174498297502e-17, genus=Cellulosimicrobium\n",
      "rho=0.7658662776556381, p=2.248451782588131e-30, genus=Oerskovia\n",
      "rho=0.8273007858537383, p=3.7898192379374016e-39, genus=Citrobacter\n",
      "rho=0.5607135760224499, p=7.032389001765838e-14, genus=Curtobacterium\n",
      "(8,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Cellulosimicrobium', 'Oerskovia', 'Citrobacter', 'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.6816394068743169, p=5.78624915635271e-22, genus=Stenotrophomonas\n",
      "rho=0.6999415405458863, p=1.5338429614803963e-23, genus=Rhodococcus\n",
      "rho=0.7613764094445883, p=7.701781555037432e-30, genus=Dermabacter\n",
      "rho=0.6882742020109265, p=1.6006239169010812e-22, genus=Morganella\n",
      "rho=0.6218839515316383, p=1.5703793923412405e-17, genus=Cellulosimicrobium\n",
      "rho=0.755037075878052, p=4.186129688798015e-29, genus=Oerskovia\n",
      "rho=0.7235701464283291, p=9.217095166991496e-26, genus=Citrobacter\n",
      "rho=0.38792837222708243, p=8.590575659663717e-07, genus=Curtobacterium\n",
      "(8,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Cellulosimicrobium', 'Oerskovia', 'Citrobacter', 'Curtobacterium'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rho=0.6816394068743169, p=5.78624915635271e-22, genus=Stenotrophomonas\n",
      "rho=0.6999415405458863, p=1.5338429614803963e-23, genus=Rhodococcus\n",
      "rho=0.7613764094445883, p=7.701781555037432e-30, genus=Dermabacter\n",
      "rho=0.6882742020109265, p=1.6006239169010812e-22, genus=Morganella\n",
      "rho=0.6218839515316383, p=1.5703793923412405e-17, genus=Cellulosimicrobium\n",
      "rho=0.755037075878052, p=4.186129688798015e-29, genus=Oerskovia\n",
      "rho=0.7235701464283291, p=9.217095166991496e-26, genus=Citrobacter\n",
      "rho=0.38792837222708243, p=8.590575659663717e-07, genus=Curtobacterium\n",
      "(8,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Cellulosimicrobium', 'Oerskovia', 'Citrobacter', 'Curtobacterium'],\n",
      "      dtype='object')\n",
      "rho=0.6816394068743169, p=5.78624915635271e-22, genus=Stenotrophomonas\n",
      "rho=0.6999415405458863, p=1.5338429614803963e-23, genus=Rhodococcus\n",
      "rho=0.7613764094445883, p=7.701781555037432e-30, genus=Dermabacter\n",
      "rho=0.6882742020109265, p=1.6006239169010812e-22, genus=Morganella\n",
      "rho=0.6218839515316383, p=1.5703793923412405e-17, genus=Cellulosimicrobium\n",
      "rho=0.755037075878052, p=4.186129688798015e-29, genus=Oerskovia\n",
      "rho=0.7235701464283291, p=9.217095166991496e-26, genus=Citrobacter\n",
      "rho=0.38792837222708243, p=8.590575659663717e-07, genus=Curtobacterium\n",
      "(8,) Index(['Stenotrophomonas', 'Rhodococcus', 'Dermabacter', 'Morganella',\n",
      "       'Cellulosimicrobium', 'Oerskovia', 'Citrobacter', 'Curtobacterium'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Decontam using raw_params\n",
    "for _ in range(10):\n",
    "    genera_new = decontam(other_X.loc[:, genera_new], other_y, raw_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Decontam + pathogens\n",
    "other_X_raw_CR = other_X[genera_new]\n",
    "\n",
    "# Normalise RA\n",
    "other_X_RA_CR = other_X_raw_CR.apply(func=lambda x: x / x.sum(), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Number of features before and after decontamination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neat (151, 685)\n",
      "CR (151, 8)\n"
     ]
    }
   ],
   "source": [
    "print('Neat', other_X.shape)\n",
    "print('CR', other_X_raw_CR.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Optimise and evaluate decontaminated models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'subsample': 0.6, 'scale_pos_weight': 0.336283185840708, 'n_estimators': 120, 'max_depth': 2, 'gamma': 1.3888888888888888, 'colsample_bytree': 0.38421052631578945}\n",
      "{'subsample': 0.6, 'scale_pos_weight': 0.336283185840708, 'n_estimators': 190, 'max_depth': 3, 'gamma': 0.42222222222222217, 'colsample_bytree': 0.9526315789473684}\n"
     ]
    }
   ],
   "source": [
    "raw_CR_results, raw_CR_params = optimise_evaluate(other_X_raw_CR, other_y)\n",
    "\n",
    "RA_CR_results, RA_CR_params = optimise_evaluate(other_X_RA_CR, other_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimate test error (hold-out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_error(param_dict, x_train, y_train, x_test, y_test):\n",
    "    model = XGBClassifier(**param_dict)\n",
    "    model.fit(x_train, y_train)\n",
    "    \n",
    "    y_pred = model.predict(x_test)\n",
    "    y_score = model.predict_proba(x_test)[:, 1]\n",
    "    \n",
    "    precision = precision_score(y_true=y_test, y_pred=y_pred)\n",
    "    recall = recall_score(y_true=y_test, y_pred=y_pred)\n",
    "    F1 = f1_score(y_true=y_test, y_pred=y_pred)\n",
    "    auroc = roc_auc_score(y_true=y_test, y_score=y_score)\n",
    "    auprc = average_precision_score(y_true=y_test, y_score=y_score)\n",
    "    \n",
    "    return model, pd.Series({'external_test_precision': precision,\n",
    "                             'external_test_recall': recall,\n",
    "                             'external_test_F1': F1,\n",
    "                             'external_test_AUROC': auroc,\n",
    "                             'external_test_AUPRC': auprc})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train on Grumaz + Kapusta, test on holdout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "holdout_X_raw_CR = holdout_X.loc[:, other_X_raw_CR.columns]\n",
    "holdout_X_RA_CR = holdout_X_raw_CR.apply(func=lambda x: x / x.sum(), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Before decontamination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, external_error_raw = estimate_error(raw_params, other_X, other_y, holdout_X, holdout_y)\n",
    "_, external_error_RA = estimate_error(RA_params, other_X_RA, other_y, holdout_X_RA, holdout_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### After decontamination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_CR_model, external_error_raw_CR = estimate_error(raw_CR_params, other_X_raw_CR, other_y, holdout_X_raw_CR, holdout_y)\n",
    "_, external_error_RA_CR = estimate_error(RA_CR_params, other_X_RA_CR, other_y, holdout_X_RA_CR, holdout_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Combine all results\n",
    "metric_df = pd.DataFrame({'Raw': raw_results, 'RA': RA_results, \n",
    "                          'Raw CR': raw_CR_results, 'RA CR': RA_CR_results}).round(3).T\n",
    "\n",
    "ext_metric_df = pd.DataFrame({'Raw': external_error_raw, 'RA': external_error_RA, \n",
    "                              'Raw CR': external_error_raw_CR, 'RA CR': external_error_RA_CR}).round(3).T\n",
    "\n",
    "final_results = pd.concat([metric_df, ext_metric_df], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SHAP summary plots for dirty and decontaminated models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAFECAYAAADFiVIeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd5hVxfnHP3Nu2Q4sS+8gIIKo6Ch2bNiJsWLDFltMjMZE/SUx9phYY0liN6hYY0dEsIsoyoAdAeltqdvrLWd+f8y5u5fdBe6y7e7d+TzPfe45Z8qZc8v7nXmnCa01FovFYunYOG1dAIvFYrG0PVYMLBaLxWLFwGKxWCxWDCwWi8WCFQOLxWKxYMXAYrFYLFgxsFgslhZBCLFCCLF7nWtKCHGYEOJWIcTEBPK4WQhxT8uVshZ/a9zEYrFYLLVorW9s6zLUxbYMLBaLpZURQkwWQvzWO+4shHhVCLFQCPGBEOKZOq2BvkKId7zwaUKIzJYok20ZWCwWS8vxihCiKu58eANxbgQKtdYjhBBdgXnAq3HhEtgXKAZmAOcAjzd3Qa0YWJqKXc+khZk6dSoAEyZMaOOSpDwi8Zin1P/d69caSn+a1vqHmmRCqAbiHA5cCaC1LhBCvFEnfIbWushL/yWwS8LlbATWTWSxWCxti2D7lar4lkWUFqrEWzGwWCyWRiMaeO00HwHnAwghcoGTmlq6ncGKgcVisTSaZhWDW4EeQogfgSnAbEz/QKti+wwsFoul0ezY+GutBzVwTXqHH8ddLgfO0lpXCSE6AZ8BT3vxb66Tfqvz5sSKgcVisTSaJrUE6pILTBdC+IB04Hmt9fvNeYNEsGJgsVgsbYjWeiOwT1uXw4qBxWKxNJpmbRkkBVYMLBaLpdFYMbBYLBaLFQOLxWKxWDGwWCwWC1YMLBaLxYIVA4vFYrGgGxCD9i4PdjkKi8VisdiWgcVisTSe9t4OqI8VA4vFYmkkqegmsmJgsVgsjaa9m/762D4Di8XSpqws1hw+JczQ/4R47OsImyvs5nltgRUDi8XSpvxuZphPVrosLYLLpkXo/kCIX74ewdXJLArNup9BUmDdRBaLpU1ZvslFCweiLkQ1RDRvfutybVaUTL9Dt0zBFfv6CPiSx+A21GfQ3rFiYLFY2pS9sqN8X+SAu3VL4L7PXcAFYNFml/+cGGyD0m0LKwYWi8XSLJRWuqzeFGXXPAfWAEKA5xpytObw/EKiQjCvWw5Pfw+PL4qQLlwqKl36d4INxZoeWYLXJgbZp0/reryT2YG1s1gxsFgsrc7azVEuuK+Qn6ocygM+nOw0XM8NlB6JIjeVkO5qQDOipIK52V2gMkJZeQSAlRUCMhxWlcF5b4T48Yr0Vn4C2zKwWCyWnUJrzeszy1i6MkS5cJjrBinuHDDuIe3Sv7CavlUh8qrCdRIC1cZdhE+YfgVXQ5ULfocFm2DfyWH84Sg90jXn7hXg9NEta9psn0EHQ0p5M3CwUuqoFsh7ALAAGK6UWielvAC4QSk11AufDESUUhc3970tltbEdTUbyzRfqXL+82oJhX4/WzKDFOemGdcQgE+wW3EBAc//ooEoEBGCnzpn1mbmeGLgE+B3aiKrFRGojgIwdUGU/r/OYP8BvhZ8KisGrYKU8mPgfaXU7c2Y52SSyLgqpVYB2W1dDoulJfk63+W05ytZlh+mmy9KP7+flV2zKMys0xkc1RT4fPSMGIMeEoINwQDfd8oAx2f6EkScAQ766nU4x9DApyuiLSoGtmVgsbQC4dUlFD/8LU5eBrlXjkEEm/CnXrwOnnwfBnSHy48G3/bzChWHWPL4YgCGndKLwH9n8m1lHitHjGKXvgGiKp/SgjB6QA7Dj+1Nj906N5jPlqcWULWwkNyzhpM5pvvOl78d82W+5pAXooR1GnQPsGVjOblBf30h0BoiLvM7Z7JvSQUBF8p9DlEBICDiuYjSfBB0zMvvQJURDgTgF1BNzfnbi6Ncd2hLPp0VgzZFSnkm8CdgMFAOvAVco5Qq98KzgZuBU4DuwCrgMuBA4Jy4PAA6K6WiUspfA1cDvYCfgGuVUrPibiuklP8EzgMqgX8ppf7h5ZUJTPHyzwSWANcrpd6LK/M44HZgFGac3FSl1IVSykHAcqC/UmpNAs+ugUOUUp9554dhWk/+uOe6CegHVADTlVIX7CjfFmNTMVz5BOQXwvUnwxeL4KMf4Oi94MYztplMh6KU7HUnWQVrqCKHjT9tpucTx8I3y3HPeRCdX4J79Fj8/70AkeEZlZ/WwB8mm3Hqd02CPQfD85/h3jed8q+3sMHtRQ9m0+meacy55CKWzSuh38bljBgGM3sfgIvg6F/1p9dHc/j+r3NZ7vSkILML77+6BifchYJOOZQu3kL5/MUEwlG6VxcytGI1G/6Wy6yTDmZtZRrB3CBj+lUy9t032VjajbVfm7/W+oe+Z9TCc0gfmNMKH3rysGRliKv+FyKM17Hrc9BdM1jueEY0VqnXnv9fg5sR5Ce/w4DiSnwaVmSlQThi4vodcB0QnmvI70CmMGlDUSMMnYNQEQEBs/IFuz0c4pHj/Xy6JMKMnyMcNsTHbeODCNF0Q25bBm1PMXA2xmgPwYjBDRiBAHgS6AMcCawAhgJaKXWXlHIkddxEUsqzgNuAE4B5wPnAu1LKkUqplV60Q4H3gN7AaGC6lHKlUuoFzAzu17x0VRhReVVKuYtSapOUcg9gBnA5EIu/f3N/KJ4oPQsco5T6UEqZBezd3PdpFL99HF7+3Bx/sQjCXi1u9kIY0RfOOKjBZNHX5pJX8BMAGZRS8M7nwLHo427DWV8EgHjpfSJDehK44yST6PS74cfV5njRWph5E0z6N46ryQEqyWA5I0kv2cCs6aWAYC1DmLssQkl+KQDPL/uJs994jW/7HG7ycQRONYBD16ISnKgmEI4SjIbYs3gpDpqcLeupnPYl3405CAoryV8Gvb4voaiiE7G/llMV4espKzngL7s320fbHrj1wc1U6SD08cRAa0jzE4kdh11AGNePT4B2wdWUdErjhy7pJnxzRW2G4ShkBkz82ChSR5hXVRQiGqoi5twnwOewsBCOey5EZakZgTR7pcuwPIfz9wm04ifRfmhXYqCUmh53ukRK+R9MjR0pZQ/gDGB3pdRyL87PO8jyQuBRpdSX3vmTUsqLMYLzd+9aPnCnUkoD86SUj3npXlBKlWFaBjHullJeD+wLvIMRgalKqclxcT5K+IEbRxgYIaX8RilVAMzaUYLmoLS0lJycnHrHkdWba39cMSGIsa5gm2l91ZVbRQ32Dxjjsbm05pogSnjlZmJ/ab22oKaepvMLEeuLt/InBwjh4qMkuHXtvDKQUXNcVuJS7qsdnqidrWt+Ub9DVXqA7LIKnLhR5pmhqq0/j7QcqvwQ2GQcCeUZaZR177zdzyqR42Cw1rXSlHxa4zgzK5uiUpfO/nCtrz++Ni4E2eEIZYE4o+wXpt3s8yx90Gc+wPhuAb9j8tOiNt8aYfHiCkwHs2PCK0Nb9yusK9XbLX+ipGLLoF2tTSSlHC+lnCWl3CSlLAHuxLiDAAZ574sbkWV/YFmda0u96zFWekIQYwXGFYOUMkNK+ZCUcpmUskRKWQTk1ilTY8qzUyilKoDjgWOBpVLKeVLKs1v6vsBWf6L4Y/+fT4M078/+y/1gUA9zPLQ3nH3INtOKk8eiRw8EwM3NIXvKr8yf/k+nAOb/HsnoSvofxtekFTedUWNsxI1nwIHDYfxoAKL42ERfctnA7psX0jXdDFvMqSrloPx5NTZq3Jm9GLBnNt2qjFA5rkvEMX+P9EglRy37iKM2z2BE+Q9Ehalpun4f3/YdivCEp4+vlOGbljC6cgFfHb8nn4/Zlc9+uR/7/bLndj+rRI5DodBOp23tY58jmHhCDms6Z24tAh6Z1RFOXLaerpXeM0VdY9SDdcxRRm1dtW8kYox8VHs/AtPPQEnICL+vjuB438kFe/sZlmeuD8oVnL+3f7vlTxy7NlGbIaUMAm8A1wFPKaUqpZS/Bf7oRVnhvQ/DDNmsi9vAtdWY/od4hgBT484HSilFnCAMwsyXBLgGGIfnllJKaSnlZmp/GSu88jQH5UBW3Hmf+ECl1MfAx1JKH/ALjLvqS6XU0ma6f+M4UcLqx6CgDHbtCxXVsGIjDO4BGWnbTtcpE6HugqUbcPrnQbapvYtbz0Kfdxh6TRH+MYMQnWtr9Vw9AU49wBiVmOhM/xP8nI972xsMeOtr0kd1g3/dyPmjB1O0upxO5YUEBx3O3iE/rgu5PdPgrL9y4k/rKImkkTEoh81XPAcf/UjPkZ3x9/PDljIY0x1960Tczp0RA/OYEPVTXREFn0NuryD+5bsS7JrNxV1y2LIhRJduQdLS21Wdq1k456TOPFYcYtlG74LWdCkLURV0OGDNFtKjmhOWr6c46OfTLp0oygxAWsbWo4a6ZABVpDmwXqSBDyMEoWhtqyGia/KP5x+HO5w6MsDQroLKsJ/lBZpBuYLMYPMY7VRsGSSzGPillPHTCv1AOlDoCcFI4LexQKXURinlK8B/vDH7K4FdvLAlwHpgfymlo5SKCcNk4AEp5VvAfOBcYC+MmyhGb+BarxN5d+ASjAgAdMKMYdgCBD0XUZe4tI8CX0opJwEvYX7OYz3D3VgUcL6U8iOMEMTKgJSyJ3AwpkO52GuhgBmq3XZ072xeAJlpMLL/9uPHCAZgt371LouhvRFDezecpn+3rc99DozoS+C53xDvIfYD3YbmAKY2uNVYIL8PZ3T/mi+w7wsNj0IWmC8SzA9gK3btW3Ofnv1ae1ZscnH1AT4+fdMY6QzXZVBhOSEcSgIBoIpVwQBf5WRTiYDKCKR7Pv/40WNd0s0goYoIhOMMfuzQwVTzXExLwWshnLtngL45xmBnBAQjeza38U49MUjmKstNmNE7sVcpcAtwl5SyDPg38HydNBcB3wCfePHfxIwSAngCU7PeIqUsklL6lFLPe3lOwRj0K4DjlVIr4vKchRGE9cDbwANx970PKALWYdxLFdS2UFBKfYtx3/wa2IgZ3TRpJz+P32I6xAuAlzFCFsMBfgOskFKWYj6b8+s8h8XSqpw8zMc7pzjccqDDp+cH+fWZXSjMCrK4cxav98rjky6dqPTVThyjImwMekNLV2/L9jpxJkwbd9GxI3z0aeEZPBpR79XeETqp1wy3tAPsD6iFmTrVeC0nTJjQxiVpOou3aL7Kd7n6pVK2lNf56WQGwO91HDvCzCtwvHkGVVFT5RHeDOQYPkH3oMvd4wOM7OVQHhYc0l/gc3bKOCecqEL8rt7vPlM/2K4VIZndRBaLJcUYnicYnufjhS98vPOT6YgXAnTAZ4w/3gghR3j9Ad4ryzNVrq412ULwp/0EfzvU1yxzBzo6VgwsFkur89y5WfzjgyoqQrDR9fHSIrwhqJhhpkJAKNa1ZyaX7d3boWRzmCVFQNDhkH6CO8ZtZzBCC5IKbqG6WDGwWCytTpcMh3+caBagG/qfUG1tv67zJep61xxu2t/huF0yeO3HCEGf4KTdWnIhuh1hxcBisVialX37CJYWxY8UMse/H+swdw0sKIDTRzhMGCYQQjBxj7afQWxbBhaLxdLMPHysn5d+CpsGgM8sN3H2CMGdh/kI+JLTRKWiGCTz0FKLxdIB6JIuuHIfzxS5mrOGC5470U/Al3oGN5lJTtm1WCwdigeO9nPWKBefgH1beT/jnSEVWwZWDCwWS1Kwf9/kF4FarBhYLBZLh8e2DCwWi8VixcBisVgsYN1EFovFYknJBbmsGFgslgYpC2me/kGTGYBJowT+nVv8LSWxbiKLxZKSFFRqnv5Rk5cO544SOEIw/uUIc9YCGv41D14/2c+AzqlnBHeO1PscrBhYLB2ciKs59MUoP24255+uhgeP8jFndW2c+fmaPZ8Ks+zXAXLTU88QNpZUbBm0p4G9FoulBVhfjhECbyvfyT/Cq4u8TfLiFpArqoIDn4nw8NdRwtFU9JonTipubmPFwGLp4PTKgnR/rXGPAue9o4110N5+AsIoxcICuGKGy6/ejbRRaS0thRUDi6WD43cEuWmYjWOiLoS9rSe19obNxNV6NaA1ry1sm7JaWg7bZ2CxWCir9g6EAEdDyGsZOAKjACJub2JNW+4kkAykgluoLlYMLJY2orwsykczi3AcOOLYXNLTW7+hXlileWi+pjoadzEmCD5hxMBrDdSgwZeSI+0Tx4qBxWJpEK01T82Psmiz5pw9fezZy0FrzRNfhfl5i8ukMQFG9/ZRFdb8/eUSilZUkrW5ks3rwwDM/7aSG27q1+rlPuj5KD9t0WaT+fh5BAGnVgBil2PhLhRWaob+O8S5uwtuPNSP0+H2IE69520zMZBSDgHuBA4BsoFCQAETgbOBG5RSQ1uhHJOBiFLq4pa+l6WdEYnCv96BtQVwyXgY3geWb4CHZ7AlL48pww8hLd3HeUdkcPEbIT6ftZ5fz/+Q/3XK4c3fHsObc8qZH8kGDfd+FuaIQQ7u8nI2lbj4oy77FIdrTMrSRZVcfcNqemY47H5gJ04Yl42T4CSv95ZGmb7E5eABDqcksBWk1prHv9P8b5HLT5u12XheYPoMtrpnneOo148gAJ/D0hK4ZbamIhThrvFtv/tYa5KK7aK2bBm8A8wEdgVKgL7AiSQouVLKgFIq3HLFs6QcL8yCafNwCysozfdR4O9Ht70dctIqYNxIwu8vomzuBiK79Cd36c84K1bibC4yaR+YBlcex7RZBczMHsh3A4ayeXUVAI+/U8QPgTSWPX0rfctM/Ec3bWD+ib8CYWrdrgPvrwKcTMgV4Lp0C0cYVFGFBir8PpyllehQmDkLqymvgDNPyNmq+EVVmns/jxDVcM0BfhZt1jz0VYSXfjQbx/9zTpSrx7r889jtG+a757pc/2mdWn/djWQ04MadCG2GGcUQnnAIwbQlLneNT/RLSA2sm6iZkFLmYUTgFKVUsXd5DfCIlPIA4BEgKKUs88JO9N7fBy4EbgG6AzlSygHAfcBBXpypwB+UUqXevTTwe+ACYBdM6+MSpdQSKeV1wDlevDO99J2VUlEp5a+Bq4FewE/AtUqpWXHPcArwZ2AYUAk8qZT6ixd2KnAjMAhYAdyslHo9Lu044HZgFOYvN1UpdaEXtgdwF7AP4APmKaXGSykHAcuB/kqpNV7cC4hrQUkpf+c9azeMwD6tlPrzDr6OjsFLn8HZ/wRMv2hnQLCEnLlbANAPTiMA5AKuUkToio+i2vThCKsfm8WEy+4FBKOqQjVB5TpAz+LCGiEA2G/d0trRODW1bW+Ipk+Az8f7PbowpLSKYZXVZEddCtL9lASD6EiE134IceYJWz/CaS+H+WC5sdBvL46yeAtb+/qB+7+Msk8fh3P32HYL4dkFcfXaWP+AEEYYdAN1XuH1G2xVH67bqdyxSEUxaJOhpUqpLcCPwBNSyvOklCOllMIL+wK4HFimlMr2Xh97SX3AccAYoKeUMh34EFgADAFGAv2AB+rc8lLgNKCHd9+3pJQ+pdRdwHMYoxm7V1RKeRZwG3AekAc8DrwrpRwIIKU8DngauNkLHw5M98IO8PL8Py/sz8ALUsqxXvgewAzgSaA30B94xgvrDXzivQZhhOjORD5TKeVw4B/AiUqpHIzQvJVI2qZQWlraLo5Dn/9Ur+zplNUcb+0QidKQI6DSH0QLBy0EJYFaY1viCNZm5/JD1z41114bsW+99FshBFrD0uwM0lxNtSOIOoJqv48tGWmIzoGtyh8MBlHraqrq/LixvhDEUOvc7X4mRdVxjxcz5m7MBVTbL4DAWAhB/c9DUzMUtaiqNiwZvuumHCeOaODVvmlLN9FhwDWY2vfuQJGU8iFMjXl7/F+sNSGlPA0QSqkbvbBKKeVfgc+llJcopWJ/l3uVUku8NNdh+ifGAp9v4x4XAo8qpb70zp+UUl6M6cv4O3Al8IhS6m0vvAT4LC7tq0qp6d75NCnl68BFwJcYoZuqlJocd7+PvPdJwBKl1N/jwt7fwecRI4L5RY6SUq5UShUBcxJMu9Pk5OS0i+Pg6QfDwzMhXGtBS+hOV9bh4KKDfkTITKRyCQIClyA+alsAmeEQGeEwlYEAazKDZEdddFWEUgAheHbkAdz52asArOzcnWBUEwFcIeJ87cKraWvwORBx2RT0kxepLVfY7+N3x2SSk1P79wyFQpw+0scTX5t4E3YVfLVWk1+rZ+Blf9KuznY/k77ZEdaUYIy5Sx0B0LXuoXj75jig3bgwL9DnEHJrxSAZvuumHCdKKrYM2kwMlFKbMbXmP0spM4EzMDXwtcR5K+vgAnErpjAYGCClLKoTT2Nq1Wu98xVx962QUm7CtCC2RX/gpTrXlnrXwdTaX6dh+mNcUXXT7h2X9uttpB0ELN5OubaJUmqZlPIc4NeYFtd3wK1KqZk7k1/KcfBu8PW9MOsndFWI0jUQdXMIjcsjvWQL4qARROcup+zz9YixQ8n66jtEwIXiMohEYJde9NtzMK8u2sLU1zcyqLCEkrH9GTOpL9Xfr2Z11+7c4j+WVXk92ZiZw4rBwzlzjI8pPxrj6kRccDUujjezF2Ngw2G+zEjjyLIK0jybetXJORw0uP5f89EJfk4c7hDVxuBvLIcPlrt0y4TFWzTlIThxuMPonttv8P9BOpzxllvj6dmKmF2vd12DFvUrwxoGd0k9w7gjUtE5lhRDS5VSFcBkKeWVwF7UN6YxtFIq/ntYCSxWSo3awS0GxQ484emO6aOAhoVnNUZo4hmC6Y8AIy7DtnGvbaWNidj20q7AuLMaIlYHzIq71ic+glLqNeA1KWUQ0wJ5U0qZ532+llEDYNQABNDJe8XjG9KLzhO9k7NHN5jFcSfCmP27ECoM0feoPvjSfXB4HlFX83pxBS/6x4KGu49J44+HBvnTYS5qrUtFOEBJlctf3gsTihp3TL9syCoJE45AqFOQK07IYuSQAMMGpzV4b0cIThpR657qnUNN38CxjRh3d/quDnMnCZ77Mcr9c93aWr638kRN30GsBeNiRhLFEIDfKJrQmlvGJYUZaVVsy6CZkFLmAtdhfOuLMD/BkzDuon8AxUAPKWUnpVTJdrJ6G7hdSvln4CGMwewD7BffYQv8Xkr5Maal8A9gGcZlA7Ae2F9K6SilYsIwGXhASvkWMB84FyNSZ3vh/8b0A3wEvAdkAqOVUrO9tB9IKZ/FuHiOBk7BuMUAHgW+lFJOwrQ+fMBYr19kCvAXKeX13vOEgUOVUh8opTZLKVcCF3nPOxK4BG+Mh5RyV4wIfYrp0C6mzpgQS/PQ66Ce9a75HMHHl2YybWGEHtmCQ7ya/YhuDiO61dbUJ+7u59lvI2QHBReM8VNYHGTxmgh7DAnQM7f15vXKXoIRXX1MX+yyqFDX1PJrcITXj+Cd1+k7HpqnuXCkw4RhDqN7dMRVbVJPDNrqWwxhOnNfAwqATcANwJVKqf9hOoXfA5ZLKYu80Tf18Gq8R2IM40KMAfwAY7jjecK71yZgT+CkuP6EJzC17S3evXxKqecxI5amAFuAK4DjlVIrvPtOAy4G7vDKvwg41gv7HDgfuAfTN3EXcK5Sao4X/i1wPMadsxFYhekrQCm1DiMa4zEtlw3A9XHPcT5mZFUxZgTVk3FhQeAmIB8oAn4HnKqUqmros7M0P+kBwamjAzVC0BD9uzj8eVyQ3x0QoFO6YGBPP+P3SW9VIYiRHRTMuTDAq6f4yY4vcmzGcczeNWD3/rKv4M8H+TuoEKTmqqVCp/jQMG9o6SFKqc92GNmyM6T2DygJmDrVeCcnTJjQYvfIuz9MQUXcVxnr14i5jSJ6q/6EY3YRvHt2sMXK00YkbNFXib/V+90P0H9p14rQ8Zx9FoulHsURXTvxLNrAiCJ/bAYyoKHKbdd2r8mkQkugLlYMLJYOTsTVRLWoFQMHgq4mFBt2GvMe+Lx+BAGXjenY65ZaMWiHKKVS71uzWJoZIUSNFygrIFj8Kx9ycoT88tqJaHv3hOvGOuze3ceo7vZvlWp0zN4fi8VSg98RXLpnrXH/476CPjkO/zzKj/BqwHt1hy/P9zNxpN8KAWBnIFsslpTkkfE+zhupSfPBPr2MYZu4m8NePQT5ZZoD+wn8Ca6i2hFIxVETVgwsFgsAB/atb+x3zRPsmmdFoC62z8BisVgsVgwsFovFYsXAYrFYLNg+A4vFYrEAqTB6qC5WDCwWi6WRWDeRxWKxWKwYWCyW1GLRFpcL7tzACR9+T3qaYNK/9qbnmLy2LlbSk4p9BnYGssXSgTnlLZfhC9YyZ1g/yiKC5y9XfLYmFU1d85KKS1hbMbBYOjDLijTPHLYX0/YZzj9OOpgifBzzXDXzN1hB2B5WDCwWS0rhxu2DVx3088Eeu/DkY6/ytxnlbVeodoBu4NXesWJgsXRQ5v1QhRuK1l7QmtnD+3HRRaew6dN1hKOpYOJaBtsysFgsKcErP0W5/d4fiATqjCERgsq0ALOHDyLga/8GrqWwYmCxWNo9D813+ey6qXw6YDA+18WJ+YpErUFzhcBN8S1xm0IquolabWiplPJH4Fal1Eve+bHAv4CemI3c04EDlFINbvQqpewHrAYGxzam38H9JgMRpdTFzfIALYSU8s9s57ktlubmxYUuXx8wnryyCPsv3si7w3vWizNgSyno3FScaNsspEJLoC7NJgZSyo+BA4Cwd2k98C+l1P0ASqlRdZI8CNynlPpPc5WhtZBSHga8r5Rq8uenlLqj6SWyWBJnQ5lL/6IQQwvKEUBeRYhN2Wm1EbTmiO+WIoRsszJaWp/mbhncppS6HUBKuT/wgZRygVJqZgNxhwDfNfP92w1SSgH4lFKRti6LJbWomLYEXR4i85e7IoJmr+ItZS6f3vEtvsIqBrt9CEZ9NXXbfdYWsqpLJv6qKgZsKmRTTiYDCkoQIvVqv82FbRk0AqXUHCnlAmB3YKaUcgVwA/AhsBjwedddYG/gbOBgpdRRAFLKXsBjwDhgA3BXfP5SSj/wZ+ACoAvwNXCVUuqHuGgZUspngZOATRixmuyl7wc8AewDBDHCdLVSal7cPU7x7jEMqASeBP4NTAd8UsoyL+pvlBbtr10AACAASURBVFJPSykHAPcBB3nXpwJ/UEqVevlp4GpgEjAKONxzl8U/9wrgBqXUFO98ELAc6K+UWuO5v3yYFtgpQDnwR+An4HFgBKCAc5RS67b1/VhSk8I/fkDJvV8CkH7sEHpOP5OykOaVQ6cyfM0G7j/+GNL8W29mnxaOctCiNey9aBmBqMv3/buzaGDftih+uyEV+gjq0iIdyFJKIaU8CGOYvogPU0qtU0ple6dHK6WylVKLG8jmOSAKDAAOxRj9eK4FzgOOB3oDs4D3pJSd4uKcAcwAugKXAw9LKQ/0whzgP8BAoBcwH3hNShnwnuE44GngZiAPGA5M9wzscUDUK3u2JwTpGKFbgGn1jAT6AQ/UKfevgIlANkbAdobTgFe957oNIwK3Aidj+mC0V25LB6P8fz/VHFe9uwy3LMR3m2Cvxav4fkB/onWEAGBTVoA+6zcSiLqUBwO8tt9ufNO/W2sWu91hRxPtmL9IKYswtdXPMAb9q8ZmIqXsCxwB/FEpVayUWg/cUifahcCdSqmFSqlqjDGMAifExZmjlJqilIoopd7DGNALAJRSq5RSbymlKpRSlZhWywBMKwDgSuARpdTbXvoSpdRn2yn2iYBQSt2olKpUShUCfwXOkVLG/wPvUUotVUpFvXLvDB8qpaYppVzgGSALeFYptUYpVQG8Auy7k3k3itLSUnvcCsfBYDCh+Gmyd825f0QeTnaQXoEyVnfrwsDNW6jLz12z+HxQd+44+WAePu5Anj1qPwL4WN41i4irk+LZW/M4cUQDr/ZNc7uJ/hbXZ9APeB54Cji/kfn0895Xxl1bXidOf2BZ7EQp5Xoulv5xcVbUSbMC45JCStkN49I5DONmis3F7O69DwJeb0SZBwMDPDGMR2NaHmu3UaadIT92oJSqkFJudQ2oAHKa4T47JCcnxx63wnEoFEooftYzEwjs2QNdFibnalMfGNIjh8IZp7H8svc5ccVC3h0wiGjUYUVeDvnZAQD6llbTq9J0X3WvCONzNY5IT4pnb83jREmFlkBdWrLPYI2U8mXg7zReDGKGcyCw1DseXCfO6vhrUkoHY8BXx8UZVCfNIGCNd/x3jHtprFIqX0qZA5RQK/ErqG0l1MVt4NpKYHEDo6YSSRtPGaamH6PPDuJbLDU4WUG63HhIvev77JrBPh+b0cuTp0RwZi/htDlzuP0XR4MQZFfXjmNwgP5binF1Lk7q2bxmIRX7DFpMDLwO4NOBbxub1hOSj4G7pJQXAhkYl0s8k4HrpJSfYgz39ZjnmRYXZ38p5VnAy5iO6FOB8V5YJ0wNulBKmQ3cWSf/fwMvSCk/At4DMoHRSqnZmGGzPinlYKVUrMXyNnC7N2/gIYxR7wPsp5RqTAtDAWdJKZ/bxnNbLE3iyr0dXp9RwMk/f8zt3t+h1C+ICvBpCAvYe/EqRGQgBK0aNEQqtgyau8/gr1LKMm+UzbeYUUBn72ReZwNpmJr+LIxvPJ67gReAmd59jsB0SJfExXkZ08FciBkJ9Js4v/9NQA9gC2Yk0eeYPgcAlFLTgIuBO4ACYBFwrBe2GNP5/JWUskhKOcnz1R+J6TheCBQDHwB7NfK5b/DKkQ98DLzYyPQWy3Y5Z6TDpbeO5bJTL8b1ho8WZKcx8udl9Fubzy9mzWdzpwzwp57Bay5ScQay0HbKuaVp2B9QCzN16lQAJkxovknqmys0Pf4TRWtdswzFkPUFHLZgBVtyMlnboxtfPNATf8fyEyX8sHPEo/V+9/vry9r1h2XXJrJYOiDdMgV3HioYur6ArKoQ/kiU/Nwc/nv4GBYM7kePqkhHE4JGkYpDS+22lxZLB+Xa/Xzk37yc6bsOpDAznV/MW0xJblfKsjIYs1dGWxcvqUnF5rAVA4ulAzNjz11YmJcLwJNHjGFIURl3H5fOLw6yYrA9UqElUBcrBhZLBybSK6dmaUnXcbjs0HR+eUhm2xaqHZCKYmD7DCyWDszd4/0Evf0M9qWKq460LYJESMXRRLZlYLF0YH4x0k/hUJfNpS4D8rJ3nMACJF/LQAjxR631PQ1cv0ZrfV8iediWgcXSwckMOgzIs/XCxpCEo4lu3Mb1GxLNwP4CLBaLpZ0ihDjCO/QJIQ5n67kSQ4CEV+GzYmCxWCyNJIn6CJ703tMxi4LG0Jhlc65MNCMrBhaLxdJIksAtBIDWejCAEOIZrfV5TcnLioHFYrE0kiRqGQAQLwRCCKdO2I5WSgZsB7LF0mFYV6bZXJFsZqx9kmwdyEKIvYUQXwghyjEzR8JAhJpZJDvGtgwslhQm6mrUBnjyuyiPfw+OgCePcbhgd1sPbAptbfwb4GnMnusXYZbmbzRWDCyWFMXVml+8HuWd5fHX4Dfvu1YMmkhCfpfWZSDwF92EZajtL8JiSVF+LtBbCUGMikj9a5bGoR1R79XGvA4c3ZQMbMvAYklR7p6bhPXXFEG3ue2vRzrwuhDiM8yQ0hoSHWVkxcBiSVGe/nHbYa7WOCL5LFp7IQlaAnVZ4L12GisGFksKorUmsh3vcUUYsoOtV55UQyeZg11rfUtT87BiYLGkII99u30XUbbd6L5JaF9yfX5xy1LUQ2v9YSJ5WDFoh0gpDwGmKqW6tHVZLMnJo99tf1BJOKoJJJlBa0+4yecmerLOeXcgCKzBrFG0Q6wYNBIp5cfAAUAIM8JsCzAbuF8pNa81yqCUmgVYIbA0SDiq+Xrj9uPY7oKmkYRuosHx50IIH2bF0oQXqkuyR2o33KaUylFKdQYOB1YCc6SUJ7dxuSztHa1h0VrYVFxzKVBaTfaaInDjXD+bik28OFYUa95Z5hL8Z3SHt6kM25nITSEJh5ZuhdY6CvwNuC7RNLZl0ESUUiuBG6SUvYGHpJRvAF2BfwLjMUvKzgB+r5QqAJBSrgAeA44ExgIrgEuVUp974QHgLuAcTOvjPuBS4Hal1GQp5WHA+0opvxf/SOAOYDhmCvoHwO+UUjuoH1qSCq3hrPvgpdmQHoRXr4XsdI66+GX8lRF4eyW89Sd492s49W6oDsNZh8Dzv+e2L1xunG2HkrYWSTi0tCHG04j5cbZl0Hy8CPQFdgWeA3KBkcBuQDfg2TrxLwJ+B3QG3sNMJ4/xJ+A4YH9gMNAPM8NwW1QDv8X4CUcDfYAHmvQ0ltZn0VojBABVIbjrDbhvqhECgHfmw9wlcOcbRggAXpiFXrSW2+dYIWhNkq1lIIRYLYRYFffaDPwP+L9E87Bi0Hys8d7zgGOAa5RShUqpQuAa4Hiv9RDjUaXUj0qpKPAEMFRK2dkLOw+4Sym1TClVCVzPdhReKfWZUmquUiqilFqPaVUc2byP1zClpaX2uLmOu2aj0wI118Pdc6BPbs05fh/06Gyue+j0ACIvh16N3MM+zZ8Ez5uEx+2Yc4FJca9jgT5a62cSzcC6iZqPft577DONXwhgqffeH8j3jvPjwsu99xygGNPCWBkLVEpVSik3bevGUsp9MG6iPYFMjGuqVTa0zcnJscfNddyjC+LVa02LoG8egQd/BUE/qxYvJSu/lLybzoUhvQg8cjkEA7CuAPF/p0C3Trx1sua6T11KqzVz8ne8xLKrRds/bxIeJ4qbZG4irfUnULN8dU9gQ6JLV8ewYtB8TATWAku880Fxx7GhXasTzGstcW4hKWUGxgW0LV4EXgFOV0qVSClPxKxgaGlvnCDNK45vrzoEgAkTDjIXuneGF67ZKs6ePQQzTvMBMGuN5tAXt9+J7Nr+4ybR1m6hugghcoB/Y+xQAAgLIV4Efqe1Lt5uYg/rJmoiUsr+UspbgAuAq5RSa4GZwL1Syi5SylzgXmC6Uip/O1nF8yxwrZRysJQyHfg72/+uOmFaFKVSygE0wk9oST0O6SfYa3tVB0uT0aL+q415CMjC9BlmeO+ZwIOJZmDFYOf4q5SyVEpZAnwKDAUOVEq96oWfixnfu9B7FWH6ARLl75hO5a8wI43ygXWYjuKGuBS42Lvna5iOI0sH5sjtDTcAM3LJstNoIeq92phjgUla68Va62qt9WLgQu96QogmLH9taSWklNlAITAuNvw0ibA/oBZm6lTj8ZswYULCaTaUufR6ZNsu44qrHDICti5Yh4Qt+ot9Xqr3uz9z3cQ2UwQhxApgnNZ6Zdy1QcCnWusBieRh+wySEM+1NBYzXyATM2dhJTC3LctlaT/0zHbokeaycRttSX+S+bzbG8nWZ4AZkfieEOI+jK0YCPweM58pIawYJCc+4HaMuycMKGCCUirh/UwtlvuOEJw7veGGm6s1jagIW+qQBH0EdfkbxpV8Nmae0TrMEPOnEs3AikESopTaDMgdRrRYtsP4QYJtefF8yVezbVckQR/BVnjbXT5FI4x/XazT0GJJUbplCrIC9a93TbduoqbiivqvtkAIMUEI8eg2wh4RQhyXaF5WDCyWFMURgndO8bFvL+iWDkEH+mXDnHPs376pJNFoomuAKdsImwJcm2hG1k1ksaQwh/YXfHWu/Zs3N0nUZzBSaz1rG2GzgVGJZmSrCBaLxdJIXCHqvdqIDG/2cUNkYyagJYQVA4vFYmkkSTQD+WvgtG2EnQJ8k2hGtv1osVgsjSSJRhPdAbwshMgFXsWsVtAbOBX4K2atooSwYmCxWCyNJFnEQGs9QwjxK8z6Z3fHBa0GLtZaz0w0LysGFovF0kiSqAMZrfUrwCtCiF0x+6ls0Vovamw+VgwsFoslBdgZAYjHdiBbLEnCdxtcjn8hzEkvh/m5wG5jmcwk27aXzYFtGVgsbcST8yPMWOpy2CCHK/b1c8KLIdZsjoKGhRt9LPptelsX0bINkqXPoDmxYmCxtAFv/BDmjudKcTS8+X06ryyIsmZTBKJmLaGf17lUR9JI86ee0UkFUqElUBcrBhZLG/Dyq8UMKQ0BkFcV5iN/Zwj4zD8yFEVrTVGlpmdO6hmdlCAJWgZCiCE7jgVa62WJxLNiYLG0EFprXl/oUlylOXmAy5b31pDWNY3CXrlsWR+qiZcZjULAqTUwfgeqI+RmtlHBLTskSVoGSzDL0tZdnrbuuS+RzKwYWCwtxLUzw9w7O4LQmsKZs9kcTKeoUzYbe4RZ1TUHn6vpVR4i5BNb1zQdAUEf320E2bvtym/ZNsnQZ6C1rhkAJIS4EDgKuJnazW1uxGyQlRBWDCyWFuKxL8MQduleWcXUUUP5eJe+7J9fyFd9uuJ6NcuCkkrKACKuqc85AlxTqXt7cRTZu3UG/BVXa37cDCO6wqZKKAvBPr3a3uAlK1ok3UDM24BhWutK7/xnIcRlwGJgciIZJN0TtQZSyo+llDdsJ/wcKeW3zXQvLaU8eDvhj0gp/9Uc97K0PuENFYTXltW7HnU1pZVRAAaUlLAqHZzyMMuy071dxgxbMoOUZgShKgqVUaiIGGHQ8NqCaIuXf3WJ5odNLiOfinLQC1H6PhJlxFNR5JQoZ02NkF9mt7huiCQcWuoAg+pcG0iCLiJo4ZaBlPJjYBwwUSn1ctz1scAcYKVSalBLlmFnUEo9BzzXSve6vDXuY2k+IuURRCRC8UtLyL/8PVzt0OPWA+h+w76UV7tkVYYo+2gVwzal0ylSzLT/3UuPilKuOPJcntx9HIGqMqI56bhZQcbmF7E4I50N6UGTua59LdrSss9x/SdR7pqrt3IwV8Xpz4uL4KVFUe4cJ7hiT4eMgNkjAaA0pMkJCkJRjdZ0uFFPyeAmqsM/gQ+FEP/FLEXRH7jAu54QreEm+gm4BHg57tol3vVW7yKTUjaw95PFkhjzfvMFq15agd+NIEvnM0qvJEQan9wf4MLgKNZW+Zj4zbfc88a7PN0zjf03KAQaDTw74kDAeIOCpZXsUVyO1ppNWWnmoqvBxdTxfIIImvUtVDPfVOFy11yT9/buoIHrPtFc90mUXlnw5i8d/u9TzUerNf1zYH25iffY0Q4X7N6BHA1JpgVa67uFEN8DpwNjMAvWXaS1fjfRPFpDDF4DLpdSDlFKLZNS5mBW1LsD+A2AlNIPXIdRsh7Aj8DvlFLzvPCjMIsw7QKEgG+UUkd5YXkY9RuP+YpmAL9XShV44Ssw+4IeDuwH/MorV66U8lXgaGAjcI1S6k0vzQXADUqpoVLKfYGP6jxTFnCyUuoNKeVA4EHgIKASs3Lgn5RSlXHxpecK2gWzuf0lSqklXsBkIKKUutg718AhSqnPvPPDgPeVUn7v/GNgPjAY02G0EbjUe/b7gQGYTqPzlFKl2/9qLImgVkU4+7+ldFpZyF/eWgFAxPGzKr03vctWEqQaX/ZGNpW5vPG/+zl66XfM7zuCvx3+S05ZkMfF899FAD0qS1mT0xWAvMpK+oU1H/bKNf0HngDgavAZo+oi+MfsCEe2wDO9vrjxM5zXl8PY52rTrY77dV0y0+XckaLDbKeZhC0DPMOfsPGvS2tIeRXG5RIzwmcBn2CUK8atwEnAsZiFlp4CZkgpc73wZzAGtzPQF/hbXNrngFxgJLAb0A14tk4ZLsFsD5cNvOldOx+4z8vzX8DTUsp6LRWl1FylVHbsBfwdM6TrM0/EpgHrMf65/TGicE+dbC7FrDkeE7q3pJQJ+/IaYBJwJ9AFeMl73kuBQzF+w12BK5uQvyWO379azspClwJ3669sS3ouU4adznv9xlEtAlzwzaectGgeGZEwD+9/Avmd8nhCHs+rIw/m5y69+ezF+/j62b9z2sLvuPqTb7j0lY/oUlZpZFx4I4p8jhEFnwAB8ze0TMvg+83Nm1/EhZcWdpz+hSTsM0AIsZcQ4kohxC1CiFtjr0TTt1a77nHgQs94XuqdAyClFBjDda1SaplSKqqUehIjFid40UKYWnVPpVS1UuojL20f4BhMrb5QKVWIMfrHS7nVoLzHlVJfK6V0XI39JaXUbKWUCzyGEYVh23sIKeUk4CrgOKXUZkxLY5h3/3Kl1FrgBuAi77li3KuUWuLd+zrvWcYm/vHV42Wl1BylVBSzz2lv4G6lVIHXInob2LcJ+SdMaWlpyh9Ho8aRvrZLNvMG5JAWDZHmhljTrQdlwWyWdh5CJNSXoZsKatIUZWQDUO0PcvdBZ9C/tJzeFaWMLFjPQx+8wuGL1tCttIJ7XngfnxtXS48NMxUCHEGeVz0JBoPN+1wtZLuS4ftqynGiJNEeyAAIIS7FbHN5BHA9MBr4AzA00TxaRQyUUj9gxr7+FejJ1k2Zbpga+1QpZVHsBQwB+nlxTsIY3e+llAuklFd71/t778vj8ltaJwxgRQPFqmmZKKU8zyfb2j4OKeWRwEPAhJiLx7vHxrj0sfunA90bur9SqgLYFPdsO0N8q6piG9e2+SzNSU5OTsofP3hGJwbmOuSkwbgu+RxV9AUD3VW4vtq/TzAS5Vi1idn9dqPSH2Dk5jX4XQ1as8EnCEYjNXHDfsEHY4Yz+ZixbOramV03FrItThhq7hEK1U5Sa47n2qtH443X9iq/PTNh4giRFN9XU44TJdnEAFPJPFZrfTJQ6b2fBoQTzaA15xk8BjwJ3KqUikopY9c3A+XAUUqpuQ0lVEp9C0z0atsHAzOllN8BC70ogzCuGzAiAqZHPUaTloCUUu4B/A84Xyn1ZVzQaqCHlDLTM/Kx+1d5zxVjUFxemRihWLON25Vj+iRi9GlK2S1NRw7ws/gm47HU1RcQmVDB4PcXsUTvQr7IoeuwHEZfNpA113/OVSecyrxBfRiT69L9my2UC4cKn8NXPfZg7MbvCDt+HjzwRL7aZTcAFg7oxbrunc2s43i0JqA1F+zhY/rquiVqOmePcJiywOXj1Tt27YzKgw/P8NEja2uDd9WHUR6cr+meAW+f4usw/QWQlH0GPbTWs7xjVwjhaK2nCyESHhXZmmLwAsZ4zou/qJTSUsoHgHuklBcrpX6WUmZjfO/fY4zqWcA0pdRmKWUhxrhHlFLrpJQzgXullOdjGr/3AtOVUvE15Z1GStkXeAfTofxmneCvMCJ0r5TyDxgf/m3Afz33U4zfex2/a4F/AMuAL2kYBZwvpfwIIwTXNMdzWJoHkRYgMPMq/GVVnJSdTqg8QjDL/I16XTqSr4SgXDvkpAly7ulGRbXmqAVL+ab7bnyfNwxXOHzdZ2tvZEY4SlH8tuVaQ9hltx60mIHNCAg+muijLKR55kfNbz9wGxxV9MxxDpNGNexAeOAIH3ccrLcactpRSEIxWCOEGKS1XoGZaHaSEGIzxsWeEK02FkwpVaWUet/z69flJkzH7ptSyhLgZ+DyuPJNBBZKKcuAt4CblFKfemHnAqWYVsJCoAg4rxmLPh7TaX2XlLIs7nWiUioCnIhx+azCiMOXwB/r5PEEZlTVJmBP4CTP398Qv8X4+Qoww3EnN+OzWJoJkW2Wl44JAYCT5scJ+shJM4ai2hW4PofPhw5A4xJ1/FT7fCzomoMvGubYBV9z4JIFFKT5jQDE0Br8gl26N2WMQWJkBwVXjHHI/7WP788X/OdIQbY3+HpUHvxi6PaNXlZQdDghSFLuwgygATMgZwrwIXBLohkIrTvOCABLi2B/QNvgsnejPPaNaSAuuflOQk6Qeb17cP4ZpzBzypPst8b4f57fawyX//J0b1QRxmUkBP8e73DFPj6mTp0KwIQJE1ql3BvKNcuKYc/ukBnoUIY+4Ye95+CP6/3u//jZYUnzYQkhgkBQa11/evw26ECzRCyW1uWRYxzeO9PH+2f66H7qruRVlnH0smXcP/3dGiEAOPX7b8F1txICtOaQ/tvOuyXpmSU4oI/oaELQKJKwAxkhRJ4QYpIQ4jqtdQjoJIRIeKCKXajOYmkhhBAcNcgYCf3MqQQnjCBSHaXrHiNZftJKBq823VpvjBwNjmPmGMSMihDMXguje7RV6S3bIxnmFcQjhBiHmfCqMP2td2FGYP4RSKhJacXAYmkFhM8h/czRAJwD/OHuS4lO/oryYBqT99sPfD7TOoh5KrSmoNylEeuMWVqRZGgJ1OF+YKLW+gMhRKxf9kvMXKiEsGJgsbQBE/bP4fDlh9WcHzMYvluryS93a9YpyvTbv2eykoRiMEhrHdu7INafEaIRNt72GVgsbcBhAx2eOdHHGSME9x/p8O7EAJft7TNrE0U1QsMhA+zfM1lJwj6DBUKIY+pcOwozPD8hbNXDYmkjJu3uMClupc//O9hPQSV8t1EzaQ+HffpYMUhWksD41+UPwNtCiGlAhhDiUUxfwUmJZmDFwGJJEtL8ggeOsyustweSTQy01nOEEHtiuqSewkzw3U9rva2VDuphxcBisVgaSbKJAYDWei1mFBEAQog9hBD/1Fqfnkh62w61WCyWRqJF/VdbIITIFELcJoSYKoS4TwjRSQgxRAjxOvA5Zr+ThLAtA4vFYmkkSdQy+DdmZ7MZwHGYpatHAE8Dl2itE965woqBxWKxNJIkEoNjgL201huFEA9h1kgbF7eCacJYMbBYLJZG4iaPGGRrrTcCaK3XCCHKdkYIwIqBxZJSuK7m4ZeKeV9VIbr4ufWiLuze1/7NmxvdUlvFNR6/EOJw4hbZq3uutf4woYyav2wWi6WteHFmOc98XMl3GWmMXl3FDTes5bSjsjn7rK44SbaeTnsmidxEGzFDSWNsqXOuqd3wa7tYMbBYUoin54b5qksWgU5BvneycDYU8+6MEgYMCHLoIa2yE2qHIFnEQGs9qLnyskNLLZYU4ksRhLwMwml+qgI+vuybS5XPYdG8ErRrt55oLpJwOYomY8XAYkkhirXP7FyvtVnnCPBFo2x8fRkvXqIIV25rgz1LR8eKgcWSakQ1VLlQ7ZJdUsWeS1ZR4PPxxWqXhe9taOvSpQTJMumsObFiYLGkCFURjT8SwVcZhlAUQlHKAgH8pZUMXLOZV3YZyPmf+rBb3TYdV4h6r/ZOyoqBlPIwKWWkrcuxI6SUA6SUZVLKPm1dFkv7ZvfHwuQUV+NWRKAyDJVhRHmITpXVdK+oYvimQpaF/Tzy2Ka2Lmq7JxX7DBIeTSSllMANmC3V0oD1wDvAnUqp/JYpXvIipVwB3KCUmtKUfJRSq4DsZimUpcMyY3mUpQWazhp01K3dPjPi0qkqxJZOWUQ7ZTG8tIKZX0X49WVtW972TioY/7ok1DKQUo4HPgMWAXsppToB4zBjWsc19qZSSrtOL/ZzSEk2FsGlD8O598PChFcProee+T3uXtcTDZ7D0Sf/j73/NBtdUtlg3E0V/9/encdXUZ4LHP+9OUkIkIBgQBQQUERF4dL6iLUuF9d6VdyttlqXira11rrV3msVrbWlal1qrXWr11q17bUuiGtFxVpbtY8oorggm7LIDmaBJOecuX+8EzOenCQnkJPkDM/38zmfzJl5Z+aZk+R95n3fOTMBRz2cgiJHqrjoC1+H6rehjve26c+CIZVMmP8xu1TX0Ku2YZPjMl4cu4lcLv2HIjIXeFlVv93C8l7AFOA4oCc+cZwfnvUiIjOAt4DhwIHAtcC5wHmq+mhkO/cBDap6lojci38A7EbgRKAGuFpV74iU3y/c72hgLXAbcKOqBiIyAZgOnAVcDVQAj4f7rA7X/wVwMjAQWA78RlVvjmx/OHA9sG94XO8CRwH3AkfgHyuXBP6pqoeKSDFwKXBGuM13w8/hjXB79wIl4XpHA38JP4sFwFBVXSwiVwH7qurBkThmANNV9ZrIcZ0G/AwYhH8Q9nnADcAJwGfAhar6SLbfVwezDmiAv/wDzr0T1lQ3zXPA2GGwz67wk+PhR/fBJ6vhjAPgyTd82cknwgH+2cjp1xeQ+vEjBG/Op3j9SuroxToG05NqylkO40ZS/OY1zXY97aM0Rz2aoigVMHRVDWUbkywuLaamRzE4x/brqjjnjXfot7GeZFERz4/eiakPjOykD6ag5FyjX3zc7GZ/9zc8MqagM0KbLQMRGQWMBB5spdhNwFfC1zBgFTBNRKJP8/42cAvQNyz/e2BSZD998RXZXZF1TgCmAf2Bhj7wbQAAHGdJREFUHwC3isiwsPxu+G6q64EB+Mr5POBbkfUT+Kf9jAV2BUbhK8xGc/AVfQVwNjBFRL4Wbr8X8AL+G367AJXAJUC9qk7E3xBqkqqWq+qh4fauxlfyhwFb478J+KyI9Ivs80TgmTDmi1v4PNuSACbg71C4a7i/V4HHwv1OAe4Jj8Hk25oqOOXmLyYC8Gly1iK47Rn4r2vgwZfh5Tkw6TZ45FWY8Q4c/Uuo2UiQTpM64jcw411K1i+jiCQ9+YzerOEzBrCBASTemgPraprtvmdxAM4xcP1GBtXWs1U6zTZFAA4C+LhvBQ+M2RmA4nSaVMouL91cAa7Zq9Dl0k00IPy5JNtCESnCn6VerqpLVLUGuABfSY2PFP2rqr6gqoGq1gJ3A4eIyOBw+TeBear6amSdF1T1cVVNh2e564Bx4bLvAQ+p6lRVTanq+8CtYSxRP1bV9aq6HJgMnB7GjKrer6pLw5heAJ4EDgrXOxLfGvhhuH5SVf+lqlUtfA4On7B+pKrzw5h+DyzDJ6pG/1DVv4TLa7NtK0c/UdXasPU1A1igqk+qahq4D590d9qM7eekqqrKptfWQCpNq1ZH/myirfGqDVQvW+mv/lldgyP9hWolQQMBRaQp9vPX1TSLobwESKYprfOVfFEQkHLBF/azrqwHAIu26kNZfTI/n0MMpnMVx26iXAaQGy89GAy8l2X5AKAMmN84Q1WrRWQFMBT4Vzh7YXQlVf1YRJ4DzgSuwbcSoq0C8BVpVA3+LB5gBHCgiBwXWV6Ef9xb1KLI9EL84HclsEJEzse3CIbgm4g9aWoBDQfmq2quVyRV4geCp4lItAlZEm4/GsPmSqlq9JKQWnzXEACqWuvH+8n7/QcqKipsesdBMGF3f6afzdYVcPmJcNH/woZ62GsnmLkAGpJwziGUjxwKQNGlXyN97dOkKaGIBtIk+IyB9GADZawjffA4ioYP/MIvtaKigpKaNKRgRVkJW9c1EBBQXZemr6unrriIjaXFDN5Qx4NjRvHqwEoGb9jYPT63bjidqzgOILeZDFT1QxH5CPgGvq8600qgDl85zwMQkXJ8n3m0Ys526nQHcLOIPInv9/9jO2JfBNyjqt9vo9ywxrjwFXwdsEpE9sH31x8EvKaqKRH5K039hguBESKSUNVs7erM41mFT1YHq+q/W4mnjVNIqoHeGfPsstPu7sWr4dUPYMlqmPsplCbg4LFQUgKD+0OfXnDC3r6FMGo7WL4OqjbATk2/2sQvj6PonP1IVdeTWryGdBq2em8lc2fN5MMxw9n/x2dl3fWXt3EUJRwbexQzq185266tJhlWVj2SaXo1bET79fVdREVFLC233sPNtUUmg9C5+DPe5cCtqrpURAbiB2fn47slfiYic/BdOTcA7wOvt7HdJ/GDvr8HHlbVte2I/TbgJRF5Bt8HH+DHBAao6kuRclNEZBK+9XIV8EdVTYtIHyCFT2aBiByBf1LQQ5HYrgNuEpEr8JX0nsC7YVfRp0S6YcJB618DvxKRSao6N0yK+wCzVXVpjselwM9FZA9gFvBdfKI13d1Xdm59eWUf/wIY1M+/MrgdBvh/yrFh7+mRsGha83GCL6zjHBeOd9zwWoKyIEVZEPD56EUQ0CsdsAZIFvle4e36xK8i62zpGH6EOV1aqqrP4QdaRwOzRaQKeAV/9v8ScCG+Evs3fmB1W+CoFs6oo9tN4RPBl2jeRdRWTO/g+/UvwHcnrcBf5TMgUiyFr9Rn4y+LnQ9cFC57Ft8SeR1/Vn8C8PmVTeHYx4H4rq65+Mtor8d3+4Dv2jpVRNaKyNPhvCuBqcBUEfksXO+7tOPLfao6A59MnwmPaxv8Z21Mi64/IMGZYx3JkgTbpJL0b0jSM5Vm9w0bGVO7kVG1G+mTTNEzmeTe43t0dbgFL45fOsvp0tJ8EpEzgP9R1TZOq0w3ZZeW5tm0adMAmDhxYqvlFsypYYeHiihNBxyzcAUl4f92EAQ450gHAV+prOP8m3bNe8wFKuca/XsnfdDs7/53f9m5oDNCl96OQkQqgB/iLzk1xmyGrQeVQlER9cUJXhzcn096l7GmpBgXnrUWOceQcc27pkz7xbFl0GXJQEQuwH/RaxFwZ1fFYUxc9OlfQmNDbWXPHvx9u/683bfpWoREEex5YP8uii5e0q75q9B12ZPOwm/63txmQWNMzipcQFXj/ZRTaZb07MGLQ7bmhzs0cNyE3gwdWtq1AcZEHL5XkCm2dy01Zkt02AjnvwBXF349pncJblAZ557Wn5Ejy7o2ONOtWTIwJkauPbwHPUqLoCQBQcCBQwPePaeE8tL4ncl2pTiOGXRZN5ExpuON6OdYdmEpUz9Is2ulY68hdr6XD3EYI8hkycCYmOnX03HGuETbBc0mi8ON6TJZMjDGmHaK4wCyJQNjjGknSwbGGGNszMAYYwwZT52IB0sGxsREKh2wtDqgLAEDettVRPkUh0tJM1kyMCYGXl+aZr8/p6kPn5ZxyPZp/vZ1+/fOlzh2E9npgzEFbml1wFcfbEoEAM99DC993NZzlMymiuNjLy0ZGFPgnluYJtuDQ55bZMkgX9Lh06qjr0Jn7UhjCtyQCke2x0r0sO+d5U2q8Ov+ZqxlYEyBO2Bo9vlPfNS5cWxJ4thNZC0DYwrc6U9l7w6auaKTA9mCxHEA2ZKBMQVsYzLg/vezL0t2bihblDiMEWSyZGBMAUunbZC4K6Ri0C2UycYMCoSILBSRU8Pp4SISiMiQro7LdK3ltfYvbDqGtQw6gIjMAPYG6oE0sBp4BbhZVd/owtBMzM1bm+2iUpNvNmZgWvMzVb0GQESGAWcDr4rI11X10a4NLeaSKbjlSfhkFZx9CIweCh8tg/N/DwuXQ2Uf6NMLJgqMGwF/ehkakpBIwA7bwIx34Qn1j4ssTUBlBayugeIi+M6hMHcZzFkMqRQcNR6uPhlunAZ1DbDrELjgHiCAu86FE77aqYd+zGOdujsTStmYgcmFqi4CLheRbYHfiMgg4Luq+h+NZURkR+ADYEdVXSQi2wM3AvuERaYBF6tqVVv7E5H/AG4BdgMSwKvAeao6ryOPq9u67AG4PqwV7/87zLoRxl8Ka2vCAkv8jyff8I+DbGjlbLo+BUvX+ek6fKUfdcuTMPV1WLTSv49e4v+Nmzo1GdwzO0WNNQy6hH3PwLTXn4HB+C6jHUVkz8iys4DpYSIoA14A5gA7AKOBIcCvc9xPAFwV7ms4UA3c3wHxt6mqqqrLp5OvfdAU0KrPYOb8SCLI0FoiyNWSNU3T0e96JVNQ15CXYywtLW02/7lFzb9olk13+B0VynSu4vg9A0sG+bU4/FmBTwxnAYhIAjgduCtcfiTgVHWyqm5Q1bXAFcApYdlWqerbqvqiqtap6nrgp8BXRKR3Bx9PMxUVFV0+XXzyfk0BjRkGE3bz3TeZipzvMopq65+4OMu/yN47N00P7Ns0LTtCj5K8HGN9fX2z+WePya0C6g6/o0KZzlXKuWavQmfdRPnVWCOtBu4ApovIRcBB+M/+8XD5CGB7EVmXsX4ADOLzfo7swi6n64G98Imn8ZSxEmjhFDlGvncY7DYUFq+GIwXKe8K/r4PfPg0LlsPgraFHiU8SwwbA32ZBr1KoqfNJ44MlMO9TPy7QswQSRVBSAgePga/sDH/+B7y3BKo3wDf3g4l7+jGGugY4ejw8/SZsrIdj9+rUwz5wWIJLJcl12qm7NcTzOxyWDPLrJHxF/oGqBiIyDzgROBa4V1UbwnKLgA9VdbdN3M/twFJgrKquFpHdgdkQw1Guluyf8dH1LoNLj81e9tT//OJ7Gdn6ti8+uvm8iZEev6P2bL68k1wyPsF1agMHnS0OLYFMlgzyQESGApOAM4CTVLXxTP1O4GJgF+BHkVWeAK4RkcuA3+D7/LcDxud4JVIfYC6wTkQqgas74jhM9zegV/wqpUKQjOHHbmMGHecKEakSkc+AvwMjga+q6sORMg/gu4ReUdW5jTNVtRbfdTQaeB9YDzwPjMtx3xcC+wGfAS/jk4sxJk+SuGavQmctgw6gqhNyLFoNrKJp4Di6jU+AU1vZx/DI9EIiXUCq+k9gTMYq9+QYkylwFcVQFcdO7G6sofDr/masZdC5TgFKgb92dSAmPm49OPv8MnueQd40ONfsVeisZdBJRGQl/iKEs1S1vq3yxuRqQwutgl36dW4cW5KGtosUHEsGnURVB3R1DCae/rU0+/yK0uzzzearjUFLIJMlA2MK3JA+2R972cuSQd5siF8usDEDYwrdlXs7yrOc1p28SwxrrG6iHtfsVegsGRhT4EoSRaw7P8F/j4fyYigrgkm7w+m72b933rgsrwJn3UTGxECiyDFl/2Km7N/VkZhCZcnAGGPaK4YDyNaONMYYYy0DY4xptxi2DCwZGGNMe8UvF1g3kTGFLggCjnksSemNSbb7XZJVtXZL6/yL3+VElgyMKXDXvpZi6kfQkIZlNTDgtoC5a9JdHVa8xS8XWDIwptBd/+/m8770hzRrN+b2jGSzCSwZGGO6m5ostz2sScGsFZYM8id+2cAGkI0pcIkiIMswQf8ySwZ5U/h1fzPWMjCmwPVp4ZRuUG9LBvkTv5aBJQNjCtjGZMCqFp6O8fVpnRvLFiV+ucC6iYwpZPfPSZNsoQHwyuLOjWWLEoPKP5O1DHIgIkkRmdDFMcwQkcsj7wMR2bcrYzJd75evtdwVZI9Fzqf4NQ0KumUgInsAlwH7Ab3wD5t/A/itqr7QlbEZ0xnmrW95mZ3p5VHh1/3NFGwyEJFDgGnALcCFwCdAOXAocCzQ7mQgIiWq2qmPNxWRBBCoqn1LqL1eeQ8WroDD94B+5dnLNCThsdehZykcKU3zkym4/Vn4v1fgvcXQqwec+zU452vw/mI47y7oXQYn7Qs7bQsfLoGfPAjJNBy1p9/vopVQXgbraqC+ASr7wshBsMeOsOsQOOxLUNmncz6LLNLA6toUW/dKdFkMsWX3JupWfgfcr6qXRuZVAQ+HL0TkZOB/gBFADfA4cJGq1oTLFwL3AAcA44GzRORJ4FZgYri9ydGdishV+JbIa8CkxlhU9cpImd2BG4A9gFrgAWCyqjaIyHBgQbjuxcCOwDARaQCuwyezMuBF4AequrytD0JEhgB3h/srBd4GLlDVN9pat2Dd+wKceauf3nkwvHG9r7wzHXcdPKF++oIj4aZv++nDr4HnZkUKVsF/PwB3TYeFKyEV5uaX32u+zT//I3tM6zfAvE/h2bf8+xHbwJu/gr692314ubjzrbZvOzHy7oC15+dl9yZmCrIlKSKj8JXon9oouh74JrAVvgLfD7g8o8zZwEX4VsVU4GZgJ2A0MBY4Gsg8tdof+BjYDp80LhORfcLYBgIvAY+Ey/cGDsEnpahvAgcCFcBK4DH8g2x3B4bhE9GDbRxfoyLgtnC9QcBM4BERKclx/cLz0D+bpj9YArMXNS+zoa4pEUTXSaVg+qzm5QHmLW9KBJtrwXJ4Y37HbCuLxz5q+9LRdfVQU2+XmJq2FWQyAAaEP5c0zhCRo0RknYisF5GNAKr6tKq+q6ppVf0IX2EelLGtu1T1TVUNgDrgFOAKVf1UVdcDP86y/w9V9XZVTarqa8BbQGMfxGnALFW9Q1XrVXUJMCWcH/XTcB/1wJfwZ/XfV9X1qloLXAocGJ71t0pVP1bVx1W1VlU34BPe9viklldVVVVdMl03Zujn08FWvWDHQc3LJ+tht6ZyyEg/v7YWRm6b/YAqenZcf3B5GdWDm7qJNud4S0tLm80/flTbgZY4SNdVd0gMW8J0zuI3flyw3USrwp9DgPcBVPVxYKvwCpuX4fNxhcnALkAP/Bn+ioxtLYxMDwjLRectyLL/ZRnva/Bn+OC7pPYRkXWR5Y7mrYvoPkaE+10uItEyG/GVeqsXCYpIJXAjMAHfCmo8tR3Q0jodpaKiokume/z8VBhcCfOX4848EAb0zV5++lVw0zQ/JnDxUU3zX/kFfOd2ePVD34IoLYF9d4ErT4K3FsC5d/nf2sFjYc+RcMx4OPXXsGSNHw8ocn5soS4JtXVQ1wC9ymBIf98ttOdIOPNAynce1iHHW19f32z+WWMTTPpb69cM3XKQ67LfUSFO5y4GtX+GQk0GHwLzgZOB6dkKiEgpvuvlUuAeVd0gIucBl2QUjfYJrATqgeHAvHDeiHbGtgiYrqpHtFEuut9F+ITSfxMHkqcA2wJ7qeoyEakAPiOOf7GNEgn4QVsfMTCoH1yb2SjDJ49HsjX6gLHD4bQDms+feUO7Qsy3Oata7/5xwHfH2eBxXsTwP6sgk4GqBiLyfWCqiKzGD/guBnoCe4XFSvEDsWvDRDAaOK+N7aZF5EHgpyLyDrABX9G2x33AxSLybXyff2NyGaWqz7S0a3xX069F5CpVXS0iA4CDVPXPOeyzD36geq2IlAPXtjNmU4D69Gh9uY0U5FEMk0GhjhkQVqz7AqPwA6bVwLvAPvhKtBr4HnCdiFQDvyW3Adkf4ruG3gdm4y9fzflpIar6Kf7qpGPwXUFrgUeBHVpZJx2WLwLeEJEq/NVKE3Lc7ZXAQGA1/kqif7YnZlOYhlQ4puzX1VFsqeI3aOCCwM4fzGaxP6A8mzbN32Ro4sSJWZcnfpUkW99izwTUXliQjf+uknON7q7c0OzvPvhpz4LOCAXbMjDGeCP7Zp9/kWSfb0w2dtpgTIGb38ItKQZvykUyJjcF3QbIzpKBMQWupYtLx1Rawz9vYng7CvtrMabAJbLUS+XFsPd28auwTP5YMjCmwO0z+IvvHTB3kiNRZMkgb+J3MZElA2MK3cNHJajs6af79YDl5xYxqNy+bJZf8csGNmZgTIGr7OVYfm6CtRuhfxm4GPZndzsx/IgtGRgTA0XOsXXPro7CFDJLBsYY014xbBnYmIExxhhrGRhjTLvFcFzGWgbGGGOsZWCMMe0Wv4aBtQyMMcZYy8AYY9ovhi0DSwbGGNNu8csGlgyMMaa94pcLbMzAGGOMtQyMMab9rGVgjDEmjiwZGGNMe+VwB2vn3ELn3O6dG9ims2RgjDHGkoExxrSbc81fOa3mTnPOzXbOve2ce9Q5NzCc/y/n3J7h9G3OuXfD6WLn3CrnXO+8HUvIkoExxrTXJjzoLOwy+iVwaBAEY4F3gN+Ei58HDgqn9wU2OOe2BfYE3guCoKYjw8/GriYym8U59yxQ2dVx5KK4uLgymUyu6uo4NoXF3imeCYLgsFwKBpcUb8r1RAcATwVBsCx8fwcwK5x+AbjMOfcAsBp4CZ8cRuATRd5ZMjCbJdd/nu5ARFRVpavj2BQWeyw4IMiY1/j+FeDLwBH4yv8l4Nv4ZDC5M4KzbiJjjOkczwOHO+cGhe/PBqYDBEFQB8wE/juc9yqwDzA2nM47axkYY0z+THfOJSPvLwOec84FwHzgO5Flz+PHCDQIgqRz7iNgQRAE9Z0RqCUDsyW5s6sD2AwWe4EJgmB4C4v+0EL5KcCUyPvD8xBWi1wQZHZhGWOM2dLYmIExxhjrJjLxISK9gP8F9gCSwCWq+kQLZc8Gfoy/wuNp4HxVTYvIBOAp4MOwaJ2q7pWneEfhuwy2xl9OeJqqzs0okwBuAQ7DX3nyS1W9u61l+dYBsV8FnAssDYu/oqrf74zYTXbWMjBxcglQpaojgYnA3SJSnllIREYAVwJ7AzuFr1MjReao6rjwlZdEELod+K2qjgJ+i7/uPNMpwMgwxr2Bq0RkeA7L8m1zYwe4L/I5WyLoYpYMTJychK+kCM9SFfivLOVOAB5T1ZWqmgbuCtftNCIyEH9d+Z/CWX8CviwiAzKKngTcpappVV0JPAacmMOy7h676WYsGZg42R5YFHn/MTB0E8qNEpGZIvKaiJze8WFCuL8lqpoCCH8uzRJva7HmerwdrSNiBzhZRN4Wkb+JyN75DNi0zcYMTMEQkZn4CiabbTpoNzOBoaq6PuxOmi4iS1R1egdt33i3Az9X1QYROQSYKiK7qurqrg5sS2XJwBQMVf1ya8tF5GNgGLAynLU98GKWoo3liJT7JNzHZ5H9LRCRx/DfBO3oZPAJMFhEEqqaCgdbt2uMI0us/47EuiiHZfm02bGr6qeNhVT1ORH5BNgdfxsG0wWsm8jEyUOE3+gUkZ3w3+Z8Jku5h4FjRGSAiBThbwvwf+F624qIC6f7A4cCb3V0oKq6ItzuN8JZ3wDeDPvWox4CzhaRorBP/pgw/raW5U1HxC4igxsLicg4YDjwQZ5DN62wloGJk+uBe0XkIyAFnKOqVQAicjWwVFVvV9X5IvIzmu758jfg/nD6eOB7ItKA//+4T1Wn5ine7wJ/EJHJwFrgtDDWp4DJqqrAH4G9gMbLNq9W1fnhdGvL8m1zY/+FiOyB/z3VA9+KthZM57NvIBtjjLFuImOMMZYMjDHGYMnAGGMMlgyMMcZgycAYYwyWDIzZojjnAufcvl0dh+l+LBkYk8E5t4Nz7iHn3KfOuWrn3CfOuUedc6Xh8jPCRxJmrtfS/FPDSrjZg82dczOcc3XhftY75950zh3fQlxTnXP3tbDsRefcre0/WmM8SwbGNPcUsAzYGajA3375WfyzDzbFOcAaYJJzLpFl+c+CICjHPxvgT8BfnHOjspS7AzjBObdVdKZzbifgP9lCHy9pOoYlA2MinHNb45PA7UEQrA+8xUEQ3B4EQd0mbG9XYD/gdGBbst9SG4AgCJLAbUACGJOlyDP4+y59K2P+OcCrQRC87Zz7hXNuftjSmOecu6CV2Jq1ZJxz9zrn7o68394591fn3LLwdadzrqL1ozaFyJKBMRFBEKwG3gXuds6d5pwb7Zzb1BYB+HslzQ6C4Al8i+OclgqG3VDfBxqAWVliSwN34++lFF3ndJpaBXOAffEtmrOBKc65r21K4M65MuCFcJs7AKOBIcCvN2V7pnuzZGBMcxOAGcAF+BuyLXfOXZGRFEY459ZFX/iz+s+Flem3gHvCWb8HDnfODcnY30/C9RcDRwPHB0HQbOwhso1dnXONT2A7FighvNFeEAT3B0GwNGzRvAA8CRzUzuNvdCT+ljWTgyDYEATBWuAK4JQWurtMAbNkYEyGIAhWBUFwWRAEXwa2Ai4FJgNnRootCIJgq+gL/0zfqBOBcppugvcUsAKYlFHu5+E2BgZB8NUgCKa1EttSfAXf2MI4B/hjEAS1AM65851zs51za8MEMxHIfAJZrkYA22ckvOfxzzMetInbNN2UJQNjWhEEQW0QBPcCbwPj2rn6d/D9/+845z7Fn/n3B87azDPrO4CTnHNfAg4g7CJyzu0DXBvutzJMUNNoeeC7GuidMW+7yPQi4MPMpBcEQVkQBEs2I37TDVkyMCbCOdfPOTfFObe7c67EOVccXuq5O/ByO7YzGv9QnGPxSaTxNR5/Vn34ZoT5LLAK/2yAfwVB8E44vw/+ltArgcA5dwStDFgDbwIDnXNHOueKnHPHAvtHlj8BlDjnLnPOVThvcFjOxIwlA2O+qB4YCDyCvxx0JXA58IMgCB5qx3a+A8wMgmBaEASfRl5vE3kIz6YIB5LvwnfjRC8nfRb/DIHX8cniBODRVrYzD/hhuI01wGFEHo4Tdj0dhB84fh9Yj+8mam8LyRQAe56BMcYYaxkYY4yxZGCMMQZLBsYYY7BkYIwxBksGxhhjsGRgjDEGSwbGGGOwZGCMMQZLBsYYY4D/B9cpb82/EKb4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "explainer_raw = shap.TreeExplainer(raw_model, feature_pertubation='interventional', model_output='probability', data=other_X)\n",
    "shap_raw = explainer_raw.shap_values(holdout_X)\n",
    "\n",
    "explainer_CR = shap.TreeExplainer(raw_CR_model, feature_pertubation='interventional', model_output='probability', data=other_X_raw_CR)\n",
    "shap_CR = explainer_CR.shap_values(holdout_X_raw_CR)\n",
    "\n",
    "shap.summary_plot(shap_raw, holdout_X, show=False, plot_size=(4, 5), color_bar_label='Read Count', max_display=other_X_raw_CR.shape[1])\n",
    "fig, ax = plt.gcf(), plt.gca()\n",
    "ax.set_xlabel('SHAP Value')\n",
    "plt.savefig(results / f'hold_{hold_out}_out_raw_shap.png', dpi=600, format='png', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAFECAYAAADftbKQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3wcxfXAv3On3lzkXsAFG9sQ6iOE0Hs1BAghEHoJEFogtFASQo2poSWU0HsLSRwwYEMwGEJ5wA/TjHtvki2rt7vb3x+zsk4nyb6TJd/pNN/P5z63uzPz9u3e3rx9783OGs/zcDgcDocjEQLJVsDhcDgc3Q9nPBwOh8ORMM54OBwOhyNhnPFwOBwOR8I44+FwOByOhHHGw+FwOBwJ44yHw+FwpADGmIXGmG1jtqkxZh9jzA3GmOPjkHG9MeaOrtOymYzNsROHw+FwdBzP8/6QbB1icZ6Hw+FwpDjGmCeMMRf4y72MMa8aY2YZY94xxjwV420MNca84Ze/bozJ6wqdnOfhcDgcqcMrxpi6qPWxbdT5A1Dmed44Y0xf4HPg1ahyAXYByoG3gF8Bj3S2os54ODY3bj6cJDB58mQAJk6cmGRN0gYTf81jWl/z3j/aa/9zz/O+Wd/UGG2jzr7AhQCe5601xvwzpvwtz/PW+e0/AUbHrWsCuLCVw+FwdC8MG74Ji/ZcwnSRk+CMh8PhcHQppo3PJvFf4FQAY0wf4KhNFdgRnPFwOByOLqXTjccNwABjzLfAM8CH2PzGZsXlPBwOh6NLic9YeJ43oo1t4i++F7W5GjjB87w6Y0wRMAN40q9/fUz7FuudiTMeDofD0aVssqcRSx9gijEmCOQAz3meN62zd7IxnPFwOByOboTneauBnZOthzMeDofD0aV0uueREjjj4XA4HF2KMx4Oh8PhSBhnPBwOh8ORMM54OBwOhyNhnPFwOBwOR8I44+FwOByOBPHaMB7pYE7c9CQOh8PhSBjneTgcDkeXkg5+Rmuc8XA4HI4uxIWtHA5HSrG6ymN2aSTZajg2SqfPqpsSOM/D4eiGXPjvOh74NIQXCPCr7YM8c2x2slVy9DCc5+FwdDOmzW7k/g/q8erCUNPIs1+GmLfWeSCpi/M8HA5HCvDCl40QDEBBJhgDdSFMevRHaUlbOY90wHkeDkc3o8EYazgw9k3WhZmsrU+2Vo72cZ6Hw+FIAZbUxHRAESjI9JKmj2PDpOsv44yHo1sRDnkEM9Ljzq2jrKzzjz/iQWMEQuBFgslVyrEB0vN6dcbDkZI0Lq5g9UPf4YU8Bp01hsYhffjrZbNZtaKRIcMy+c3tY8nK7fwOs7I6Ql6OIRhM3T98aa2BoAfVIWtAgN9OruetM/KSrJmjLdI15+GMRxyIyD7ANFV156sL8TwPb/E6lvx6GkvfXcvq4gICEY9Bd3zJR3uMZ1WfYgCWL23krp9/xoVP70R+36xO2Xck4nHF7SV8M7eR3GzD3b/vz4ihmS3qhCMeq9aGKS4Kkp2VvA4hGI6AZ6zhMIAHU+eEk6aPY2P0cOMhIqOAScCeQAFQBihwPHAicK2qbtUVSm5Ap/ewnfpNm3O/js7Hi0RYc8CjNPx3MbMZybJR/ajP8i/P7Dr6rithRZ9i8Oyddn29xxt3zuHgy7emqHd8l3F9XZgXnyyhuirMAYf1Ycz45jv1N9+v5pu5jQDU1nv87s+reeXeIRh/GFNdg8fZt5fy7bIIOVnw8MV92HZkxw3X3FUhcjINw/om7j1FPKC8AXIzICMAEQ+v0RmPVCVdPY9ERlu9AawAtgYKgd2At0hxsyoimRuv5Ug2VW/PZ830dVTQh4GmotlwAFXZuZw481XyayrJCIfJDEdozMzkyy9r+cPZs/lsetlG5UciHtdcsYj3plegWsVdNyyhfF1offkn/1fdon5tjcf8JY3r16f/Xy1fLwuTFYlAXYRz71vLuqqOPVtx7uPl7HXzOn58fRl3v1ndZp3vVob5eGEjHy1oZE5JS8NQXhOBoLGGAyBgICvINe82Eoqka3q2O9ODR1uJSDHWaByjquX+5qXAgyKyG/AgkCUiVX7ZEar6nohsC9wJ7AzUAM8Cf1DVRhEZASwATgF+DwwH/gecqqorovZ7N3Ag9oy/BVyiqmtF5H6sF7SbiFwFLFPVrUXkCSATaACOAl4EzhOR84DfAoOA74HLVfUDfz/X+7Jm+vrUAver6p9jzsPxwC1AP1+XM1W10i/bErgX2N1v/yrwe1Wt9cs94ELgNGA88BXwC+A44FIgD3hQVa+J2t/ewG3AOKzhvltVH/LL9gGmAb/agE63AL8EBgCrgPtU9S9+WRZwP/AzIAdYCVytqq+QBMqengmRII0EiXhBelfWsK7QegZjyudDwAMyMP4fzwsGyWhooN/adbx+cwWrr1nFrjsXUNW/mLVvLqPXngMZdfPOmICt/9GMCspKw2SFvaZID9eeP4+RY3P5+ckDKK8IkxUOEwoECHoewUiElatCjN7Ceher14YpCkfIjXgEPQ+v3OOE61bxyO/7M2xABo0hj/tfrWT2kkYOkByO3Se/zeNcWhbm3182G6Xb36jl5N1z6VdoDcH8NWEOfbiK2SUtDdMTJ+Rx6i7ZzFkToSHkH4Dnsf4Bj4Dhls/g5TkhfjkG3l0QYf+RAa7fN2O99wRQ2+hxyeR6vl0V4bSdMzjzx50T9nO0T4/2PFR1DfAt8HcROUVEJoiI8cv+B5wLzFfVAv/znogMAKYD/wCGYD2VA7GGIprjgb2AoUA+cENU2bNAH2ACtsPtBzzt7/cC4APgRn+fW0e1Ow54E+gP/E5ETgBuxBqGYuAR4E2/w29iL2wHOxhrdC712zURBA4CtgfGAjsCFwGISAbwOrYD3hL4CdaI3BFzrCdhO+v+QB3wrn98o4H9gMtE5Ke+zJH+MTzo63wacKuIHBePTj7fAXtgPcWz/fYH+2WnAbsA41W1CNjfr7/Z8dZWkfna/wgDYWwYZ+yyNWyzZD5HLH6LH5d+yXe9xxMOtgzx1ObkEAoGyKtvYGZGX+Y+MZvS6z6k/MPVLP7z16x4dPb6us8/U0owHFn/NzZAYwjmfFfLX29bRk5WgIJwmN6NjRSGQmRHIkz/rNkr6Ns7SHbEIyMSoVdjI31CIYrW1jH5fVvn+WnVvPJeDTPnNXLXi5XMnNfQ5rG+/kVdq22/f6lq/fKvnqluZTgAbn+3FoAVlWHAs895BPznPAzWEwHmrIMb3w/x4eIIN0wP8dzXLb2Wm99t4KFPGpmxMMzZ/6jny2Uu3OXoGImErfYB3sPevf8fsEpErmsyIm1wCvCVqj6kqg2qugy41d8ezZ9UtVRVK4DnAAEQkSHAwcClqlqmqmXYO/TDRGTwRnSdoaovqmpYVWuA04GHVPUTVQ2p6qNYL+PEqDYrgEm+rp8DD/vtorlKVatUdRXwzyZdgR8DY3xdq/1jvRY4I+b83KmqS32dXsF6Qdf7+/wK643s4tc9AfhCVR/3df4YeAg4K06dUNVnVHW5qnqq+i7WwO3vFzdgc1cTRCRDVZeoapcbj8rKytbLZdX0qy1hTVEWHhCkgWKWkV/dwILaCXxmdmdR3VgCkTCBSBgTiRABGrIyKSssJBwIEPA86rIyCdLc8VYuWAfYRHxNjdfuePvK8hAEDbFd9pKljev1XFURIWQMWZFmAxQEQpU29LWipKVRKF0XafN4X/q09dN81XXNnsiydW135kFjtQ9FgPxMG7IKR6DBH3HV5F3EHOTyCq+FDovWNBs1z4MVlV6berrl+JbjwcO0+qQDcSfMVbUUuBq4WkTysCGXR4Bl0Op/BzAS2F1E1kVtM0BshnBF1HI19i4ZbBgLbGiriXlRZdHtYlkYsz4cG76KZl7UPgAWqWr0X28hcEzUelhVSzag62pVjQ5gz8OGg/oDq/1t0TrX+G0iMduiZc5vQ+ej4tQJEbkI63EMw577XKyBBngGGIgNC44RkXeAK1R1Ll1IYWFhq2UzeiC1W23JznO/pzTYi0HhBWRRzxbAt2Z7ljKScIahz6pySof0JRww1GT74RZjCBtDn8pqhgQbqBjZHxbUkj08ny3P28avYujdO8C6tRGC4dZ/3UOOLuaTJZEWF3EQGFwcXK/nF3PKWJMZJCsSJjfSXPOA3Wxo7RcH9Ob9mTYPMm7LDHbbNqvN462oW9ti3wa4/Iii9evXHZTHOS/XtDJ0j59YAMCovkEwdk4rqn2jUxeComzIDDKyF5gcw/wyj1F9DCdvn9FCh0v3zuP12TWU1cIeI4Lsv1XzMW7oN3LLbS/HR3oYi1g6NPTUv3N+QkQuBHbAjrqKZRF2JNThHdRtif89Amjq0EbFlLWXsWx1E4k1ZtGMAiZHrW8pIibKgIzA5nXi1XWAiOT556ZJfh1QGqeMtmQeFrNtFM3HvkFEZHfs6Lj9gU9UNSwir+Bfyaoa8ssniUhvbP7jMWz4brOTfdF+eBe9SL/wGrJovjvvk7Gaj0buSGNWgP4lFaweXExjRvNlm9nQwIhDB3PsIWPJHrEfXmaQuvlV5GyZTzC/eazENdcP4w9XLaau1iMjHCFo4IIrhjBoaDb9BmQxc9JyckNhGjNsZ+oBEw9q7iS2G5XJ9O8bWZ6TTV3AMDTH48Jf9WH8uFwARg7O4KUb+1FSFmHYgCAZ7Twn8rOdsnjgnWYv5T+XFLLjls16nr1bNoeNz2TB2jDPf9HAsnKPC/fMYqdhts6I3gFOnhDm6Y+iPBQPqAux9aAgX50eJOJlsKDMY2QfQ25mSz12HBpkwZUFLK+IMKZfoF09HZ1HungascSbMO8DXIHNQfyAvVyPArYF/gyUYzvPIj/8BPAUNt9wBvZutwHbIY9V1Tc3tk9VXS4ibwN3isip2E7vTmBKU0Idm2OIZ3jwE8A9IvJv4Ats7mEHWoatBgOXi8jd/nGdjQ2TxcOnWAN3p4j8DuiNzbE8HuNZJMLzwHUicgr2/O0EnAOcF2f7IiAMlACeiBwOHAq8DCAi+2F/t5nYBH81EGpbVNeTddZu1D31CTW6kgzKycKGVyroQ2N2EDyPdX3zMETIq6sjEgjQd2wR2+87gH2OKG7xUF/+hN6t5Pfrl8V9D43mjX+soXJdiL0O7M2wETnrywflG5aEwgQ8D88YIgZ+ND53ffnpB+bzzlf1fLc0TKA4i79cWcyA3i2d6PycAPmDNxwJvvrIfAb2DvLNkhAn7pbNjm0M9x3aO8DQ3gH2GNX2QMGnJmby9Mcx4a+g4fuzmpPjEwa032H1yjX06oIHLB3t0YONB7bjH4BNfg/GdjILgQtV9WU/YTwVWCAiQeAoVZ0uIvtijcst2JDJQmzcPl5OwoZVZmF/gbeBS6LK7wYe90Njy1R1m7aEqOpzItKX5lDND8BhqrowqtoH/rGtxHoM99Ac4tkgqhoSkSOwo60W++3/AVwV32G2KXOBiByG9Q7u8/X6g6q+FKeIt7CDCz7FGvt/Aa9FlQ/EehtbYH/fT7HGKSmY3CxyP7uCjKUVfDnuWQZUL6COPEpNP3ZY/TVfbLUjZdlFhINBGgIBCgoCXHznmISeBA8GDROP69dm2a67F6IfVJATjuABPz9zIBlR06BkZhhevKqY6roIedmmxQimhI7TGM7aO3fjFTdGKGyf8wj7DwrmBglHIMPZhJQjXT0P43luXLg/VHcPVT0g2br0ADZ6wdUtrmLlU3PJWLCavtsVkHv6jsz9vo6XrpllKwQMv35kO/pv2bnTccz6ppoFs+vYdsd8ho/M2XiDJFFVF6Hw1no7LTuA5zGqMMK8S9s/H5Mn2wjtxIkTN4eKPYG4LUKNuajVNZ/n3dvtLYqbbsORcuRsUcCIa3dosW3Mrrmcdv+2rPihmpE79aJ4eCfcvccwbtt8xm3b9vMZqcTv3gmtH5oLgDG8doJ7FtaxeXHGw9FtGDqukKHjEh3pkn5UhgJgPMgM2PvfUIQGz72aJ1VJ17CVMx6Aql6fbB0cjng5ZAQ8PyvQ7H0EAnxf6iFDkqqWo12c8XA4HCnA4195LfsjYxjfP2nqODaC8zwcDkdKUJSDfZLJ+E+WN0boleX+yqlKuhoPFyh1OLoZfzs0g9yAZwfM10cYlBFmSGF6dlCO1MXdrjgc3YwhhYY1l2Tw5xlhquoN5+2STX4SX07l2DDp6nk44+FwdENyMwP8aV8XOOgeOOPhcDgcjgRxnofD4XA4EsYZD4fD4XB0gPQ0Hi5o6nB0AauX1lL+7jJCpbXJVsWRZLw2PumA8zwcjk6kLuRxzLXLOOOJD+hVU0+oVzb7fXA42SOKNt7YkZaka9jKeR4ORydyzzt1nPL0RyztW8j0CVsQLKtj6VNzkq2WI6mYNj7dH+d5OBydyLySCP/76Xj+tet4AB4+sJorM0r5XZL1ciQP53k4HI6NcuYe2by+89j16yW98nm5z+AkauRINh6m1ScdcJ6Hw9GJ7Doig1CwZUp0bUOSlHE4uhDneTgcnciSF2ex78z5ZDU2vw4+HE6iQg5HF+GMh8PRSRx/8Xy+OvdTBpfX0BgMgv+K5/nhTFZWOQvSU0nXsJUzHg5HJ/D+p+UsrTLMGL8Fq/v3YtvV6+x06QDGcPzkdBnd70gUZzw2ARG5XkSmRa2/JyLXJtA+ofobkLOniKzbRBlTROSKTdVlA/IXishJGyi/WkQmd9X+HR3jH1PX0reqhnuP+Am9wtC7rrFF+eLKJCnmSAF6+FBdERHgWmB3IBtYCbwBTFLVFV2jXueiqh8AvTdRxqGdpE5H939LMvfvaJuM0X2ZGhhMfWYGdZlBtl9ZxsqCHNbmZlGbEeTwvh5ufErPJF19zrg8DxE5EJgB/ADsoKpFwN7AGv/bsRFExIiI6z3SkMbVNfz0sn8SCti/0/sjBvD26EHM6VfEmvwcdl62gPzvS2BNBez0Oyj6FRz8JyirSrLmacTzH8DFj8K0rzZNTl0D3PoqXPYELFzdKaqla9gq3s7sr8Bzqnpl0wbf27gRwO8UrwBOAwYA3wIXqernGxMsIiOABcBwVV3qbzsNuFZVt2qnzXbAX4AdgTLgMeBWVQ2LSBZwP/AzIAfrIV2tqq+IyD7ANFXN8OU8AQSBRuAYoBq4DPgeeAQYByjwK1Vd7rd5z5dxU5T+twN7ALn+sR+pqmtExAN+C5wMbAPsKyJzgLuBA7H+61vAJaq6NuoQR4nIDGAHYBZwnqp+5u/vemAPVT3AX1/on6tn2jqfHTlGR2LMu+tLLv/VAYSDAXZaOp+C+jo+GDV+ffmHI7Zi0nO3w8ul8PUiu/Htr+DgG+DT25KkdRrx0odw4t12+YEp8Mkk2Hl0x2T95mF4/F27/PJHMOcByMrcJPXSxVjEslHPQ0TGAlsBz22g2g3AUcAhQDG2M39LRPp0hpIx+vQCpgL/BQYBhwNnAJf6VU4DdgHG+x7S/sB3GxD5c+BVoC/WGD6CPZ6jgYFYr/P6dnTJA94FVmM74X7Yjjl6ZP+ZwPFAAfAl8CzQB5gAjPfbPB0j+lzgYl+nV4A3RGRTJkfq8DF2NpWVlWm3PLU+k/n9B3LVu6/x+T1XMf3B69l69bL19SLGEKypwVtSSgt8Q5Js/bv9ss5bv41whLqPv++wzPCns5tlLS6FkooN1o+P9Mx5xBO26u9/L2urUEQMcCFwuarOV9Wwqj4KrMB27J3N4djO+SZVrVfV74FJwFl+eQO2o54gIhmqukRVN2Q83lXV11U1AjwF5ANPq+pSVa3Bdt67tNP2CKy3cbGqlqtqSFX/p6rRV9cdqjpPVcNYw3owcKmqlqlqGdboHSYi0Y8hP6qqn6tqg39stf6+OsqmHGOnUlhYmHbLEw4fCsA5H9sxIY2BIEd/8ykmErEVjWHGxAMxJ+xBC47bLSX07/bLR+0CWX4QpU8BOYf/uMMyg8dH/Ua7j4PBfTZYPx56ctiqxP8eig11xNIP21lP9sM0TWQCwzZNvTYZDixU1eh9zfO3AzyDvZu+GxgjIu8AV6jq3HbkrU/2q2qNHRdA9ACAGqC9q2UEMF9VQ+2UAyyM0R1sWCla96aypv2ub6OqnogsZtPO5aYco2Mj7H/AULafuog5xYMYUVbCdQcfz6T9fra+3EQibH/YtrD/PrD/dvCvT+HQHeH4PdoX6oif3cfDF3fAF/Nh721gi/4bb9Me1/0CfjIW1lRZoxTY9AGp6Zow36jxUNXZIjIXOAGY1kaVUmwc/YCmuHyCNGUN86O2DdlA/SXAliJiogzIKH87fkc+CZgkIr2x+Y/HgL06oNvGWAiMFJGg71m0RSRGd7BGp8mYjYopayoH1nt2WwBL25FfRfznztFF7LWslBNPvIjrp77MByMmtCgbVFbJPz+sZv/9+8Gxu9mPo3PZZgv76QwO3KFz5Piki6cRS7wJ899gPYtVwP2qulxEBmDj+fOBe4A7ROQsVZ0jIgXYIb1fbywJq6qlIrIIOENErsbmAs4G2uuMX8cmy68WkduBkcCVwEMAIrIfUA7MxIZ7qoENeQabwuvAbcDdInIdtiPfBfg2JnQFgH/e3gbuFJFTscHPO4EpMcOdzxCR14CvgUuAPH9fbaHACSLyLDaEdl3nHJojEfIDUFrYiwuOOYuC2gZMxMMLGIaWVTJ+TRXztt2kEeKObk16Go+4fDJVnYodTTQB+FpEKoEPsSOrpgN/BP4F/EtEKoA52KRvvD7fqdiYfjlwF/DoBnQpBw4CDgBWYUcrPeW3Axuyeho7CmsFsCVwTpx6JISqVgP7YUNOc7BDl2/Hhuza4ySgEjuKahawDjglps7DwL3YYzgeONw/7ra4FmtoVwDvAS904FAcm8i4PZtDJbmRCCd8vYiJs5Zy5OwVFDWEOefggiRq50gm6ZrzMJ6XrhE5R4qSthfcvvdV8F5dLgCH/7CMrcqqaQwYPho3gC9v7JdU3SZPtpMSTJw4Mal6pBFxW4DF5uZW1/wW3jXd3oK4ua0cjk7ivxcWsfDYEAd8OZeZffP555hB5K8oYeK8RclWzZFE0tXzcMbD4ehE6qvDrO7XiyUD+rCoXy/uPezH1ORs2kNmju6NMx4Oh2OjbDUhnyX9mp/nrM/KbPFmQYcjXXBzLTkcnUggaIiYlneWdRnub9azSQ9PIxZ3VTscnUxFblbziucxcXR6dh6O+EjXESIubOVwdDKvHhVYf685ON9w+z7BpOrjSC7pmvNwnofD0ckcPSZI5DKoavAoyEqPjsLRcdLFWMTijIfD0UU4w+EAZzwcDofD0QHSNefhjIfD4XB0Kc7zcDgcDkeCpGvYyo22cjgS4IVZEc55O8wrP0Q2XtnhwI22cjh6PP+ZG+aE/9gI9sMzPV41HseMbXsY7udLQiwt9zhwbAZ5LnHeo0nXnIfzPByOOHnk65bdwKRP2+4W/vJ+HbvcXcmxj1Wx6z2VNITStftwxIPzPByOHs7SmNd7ra5pu94j/1jHOUvXkul5fFhRxLcr89hxmPur9VTSxVjE4jwPhyNeYhyI9rqEHVeVk+15BIA9yiqorW3vpZgbpn5BBXMO+w8/7PNPqj9d1SEZjuTjtfFJB5zxcDjiJDZFHmjHejR6toP4riAPLSrgvHvXdGh/3x/6OpVTFlM9fTmz9vs3XiRdup2eRbqGrZzxcDjipC7Ucn1hOZz+ZohQTKdenhHksaEDmdqvDx/17cXcQDY/rI5pHM/+5lfQEAxQk5UB1Y2EKhs3RX1HkkhX4+ECsQ5HnMR6HmHgiW9gcH6YW/Zs/isty82hKrN5vSYjyKyVEbYekNj+lvfNZ11+Lp4xZNc3MqaqgcxeWRtv6Egp0tVfTHvPQ0QWishJnSxzroic1pkyHalPe3+W2z5pub44J6aD9zw+mFOb8P7K83Lw/HeD1GdnUnHmExDuWP7EkTyc55GiiMh7wG5AI/ZmcAFws6q+nEy9HD2HMHDalBBPHGr/To0mwO5r1qG9i6gPBsAY7v0gxIRh9Zy2izUsgfYSJlH0aiynLLsPAMFIiL5vf0j5Puvo9cG1tkLE94UCaX8P6EhB0uWqu1FVC4Bi4AngORHZKrkqOdKN7A28luPJb+HWj23+46A16+gXClnD4dMYCHD+i1X0/u0atjlvOb+4cw0rKyLMXB7maW1g6boI1ZUh9P11LPjBjgHec9XHDK1eRnHVOrZdVML33h7MmdGXrwc8St1uf2JN8AKqg2fA8bdDVeKejWPz4DyPboCqhkTkEeBuYAdgrl+0hYi8A+wKLAR+raofAYhIBnA1cBrQG/gSuFhVv/HLM4FJwEnYsPfdsfsVkWOBPwAjfPnXq+prUeV7AzcB2/gyJqvq6VFltwHjgBXA3ar6UFTb7fzynYEg8LmqHuiXjQBuB/YAcoFvgSNVdY2IeMCeqjrDr7sPME1VM/z1XwJ/BIYBNcAUVT0tvjPdMwlu5Fbr6hnw2aoIfRsbCRuD8bz1YSeMIbchwrErS8mPRAiXwgFLGpmVkUPYg355htPLSmlYUQfAiRcOpa5oLMvyhzJ6+VoCESvHAIGSEuaW5NPAtoDHiJe+oe/c6+DjP0NmWv2l0wKX8+gGiEgWcJ6/Ojuq6AzgIqAXMBV4MqrscuAU4DBgMPABMFVEivzyq4AjgJ8CI7EGYsuofe4GPOvXK8YaoudFZFe/fDvgLeBRX/5w4Cm/bCTwJvCg3/Y04FYROc4vHwxM9z8jgEFYQ4aI5AHvAquxhqcfcBnQEMd5ygOeBs5X1UJglK+fYwPEjrZqi9fmeJRkZRIxplWnsWVtHfl+qCkIDFhXR9ivVFrjMbOi+Y70q/9VsDR/EECzAcJ2RCEyaSDP32JYxwD4Yj4sWN2xA3N0KenqeaSL8bhGRNYBtdg7/LNUdWZU+UOq+q2qhoG/A1uJSC+/7HRgkqrOUtV64AZsGPtwv/wUv3yuqtZiO+jofuF04FVVnaKqIVV9HXgNa7AAzsV6Gk+oar2q1qrqf/2yE4AvVPVxv+3HwEPAWX75ycBcVb1VVatVtUFVp/llR2C9jYtVtdxv/z9VjXkOul0agXEi0teX/UGc7TaJysrKbrscz59lhwGQ1xjivT69IKrTL2oMUdgYIjrdvSY7c/1yZgCGBppLh4/OYYsq+3zIsmLC9/oAACAASURBVOJCarIyaAwYVvTJBwwBmi1ZHhUwuA+VRc2J+mSfq56yHB+mjU/3J12Mx82q2ht79/0GsF9M+Yqo5Wr/u9D/Hg7MbypU1Qg29DTc3zTMX28qr8be7TfRor3PvKj2I2jpBUWzKW1HAPNVNeEHCFS1ButpHQLME5HPReTEROV0hMLCwm67vLGI0NFbwVvHBqnMzOAnZeUMrLdOYFYkwo7rKhhV4FE5MJfq4hxqt+/Lg78fyNMn5nHxnlm8dU4+N1w/nL2P6MuxZw7ioJ/3Z2TZan60bCljS1axujib0n5Qmxdi+B+3Y8y2JQxgMcPNLAZOHADTb6RwUL+UOVc9ZTke0tXzSKsAqaqWichZ2A7xKFX9VxzNlmDDUQCISADbMS/xNy3z15vK84HoEfst2vuMimq/EBizgX0ftpG2P2+n7UJgpIgEfY8qlmogP2p9SHShqr4HvCciQeBI4FUR+URV57Wzvx5PaAOzsP9mB3jgAPt3+qh3ESetKOG4FSWsycqkIBSmOmh48PYx5OS1zLrvNgJO2rnJY8jk6NMHry+r9PqQUWdljllRhslewYKtfkSfP+6PuWYf8r9eBEOLYWDvzjxMRyeTrjmPtDIeAKq6VkTuAm4RkclxNHkCuEJE3sd2yFdiz8vrfvnTwOX+kODl2OS1iWn/jog8DUwDDgKOAfbxyx8CPhGRk4EXseHuXf3O+3ngOhE5BXgO2Ak4h+a8zTPYkNyVwH3YUNNeqvqOr99twN0ich1QBewCfOuHrhQ4VUT+izUclzYpLCIDsUn2aapa7of8ANxDBBugsZ2zk2GaDQdAKBhY/8ca0NBIBPjJUf1bGY6NUU92877JolfvLA56ayLGGOsG7TQ6wSNwJIN08TRiSZewVSz3YJPTp8RR93ZsJ/42sAob8jpIVSv88luxCe+Psc+QLAYWNTX2R22dCtwBlGE79JP8/AWq+hXWuzgPG+5ajM1loKoL/LILgDVYQ/UHVX3JL1+ONUIHAkt9/a70y6p9XYcDc/z2twNNgfQLgK2AtcBLWCPXRAA4H1goIpXAA8CpqrowjvPliOHEcS3XI57Hiqh8xvzcbEbvmrh30Bj1rGGGaWDYN1eSPbRX+w0cKUm6ToxoPC9dDsXRTei2F9y4R0P8UNZy2zbFMOOEIL1zmu8u8y8upSEQYGx1LXWBAKVZmSy5rZiinMTuQF/Z4nH6VDaSFQ6zrFcBP//qWDL65m28YRtMnmyd8IkTJ3aovaMVcf+YH5uHWl3zP/HO6fbuSNqFrRyOzcU2feGb01v/hTwgZAzfFdqU0y9GRxI2HABbZ62mLNMQysrgR5WLIX+zjGlwdDIubOVw9HAyYv4t4XZ8qMJwpMUw3T8eWdCh/Q148SwaB/UjUpBD1iO/JmNDj7g7UpZ0DVs5z8PhiJOsmL67vp3RV0Umsn4st/E8OvoK84E7FzNw5hkbr+hIaZzn4XD0cPJjbrV6Z7dd7/zREQbX1VPUGOLocCWjBme2XdHRI3DPeTgcPZxjxxpmLG8OOvx6u7Y7gQvOH8SOb5VRWxNm94OGxDWDriN9SZcwVSzOeDgccXLRzgEWVoSZtggOGwXnbN+2456RGWDvI4o3s3aOVCXVPA1jzGWe593RxvZLPc+7K145zng4HHESMIa/7Of+Mo7ESDXjgZ0BvJXxAK4FnPFwOBwORzPGmKY5/4LGmH1p+azKKCChGR+d8XA4HI4uJIVyHk2vXcgBHova7gErgQsTEeaMh8PhcHQhqRK28jxvJIAx5inP8+KZummDOOPhcDgcXUgKeR4ARBsOY0wgpmwDc0e3xBkPhyOJ3HnVD5RMW0GvUJhTJv+UocNzk62So5NJFc+jCWPMTtjJULfDhrDA5j887KzfceEeEnQ4ksSU/5bx4XcN7DhnFUOXlPH0wZvlZY6OzUwKPiT4JPBfQLCJ8lHYdxKNSkSI8zwcjiTx0POlnPH2V9RkZ9K3sjbZ6ji6iLjjQJuPLYFrvE2cUt0ZD4cjSXzcpy+rTtyDjycMo29FDb995eNkq+ToArzUm2HgNexL697aFCHOeDgcScIzho8nDANgbVEejx26I9clWSdH5+OlnO0gB3jNGDMDO0R3PYmMwnLGw+FIEv1q69fPvgswflEJny8sYucR7cy46OiWpKDn8Z3/2SSc8XA4ksQBXy3gpzOX8NYuoxm+uoIz3vyKa8f1YspNQ5OtmqMT8VJsWJLneX/qDDnOeDgcSWJ1TjY7zVtNr6o6Bq2tJjMU5v38vslWy9HJeMHU8jyipilphed578YrxxmPFEFETgOuVdWtukD21cBuqupeYJ0ifDy3gVf23pYXDrDD6k9++ysCXoRIqj1R5thkIqkXtno0Zr0/kAUsJYHhus54tIOI7Ab8EdgN++DMD8C9qvpkUhXrAKp6S7J1cLTkr9NrCWXmr1+fuvNoTnz3W47+ZDawffIUc3Q6KRi2Ghm9bowJYmfUTWhixBQ7rNRARA7CPkTzP6wlHgBMAv4iIh2KF4qIe52cYz3z61o+yDustMJ+19dRUuPcjw7z369h72ug90lQeCLc+FKyNcILmFafVMLzvDBwM3BFIu2c59E2DwDPq2q0oXhJRPKAv4vI46q6UETOBi4GhgPzgStV9W0AEbke2Av4AjgZ+EJEjgLuB36GHS63ErhaVV+JVUBEDgEeB85W1f+IyLHYefhHAAuB61X1NRHJAJYA56rqv6LaPwmEVfUMX5c9VPUAv+xi4DxgKFAGPIsNmYU38bw54iRrbS3k5mAiEXaevZKL/vEJn48ZxL9HDuW+h8O8emSAw0a5e7uEuO45uCnmr/SHF+CzufDvq5OjEyk5VLctDiTB5xmd8YhBRMYCWwHntlH8HDZeeKCIGKylPhb4GjgE+IeI7KCqc/36ewGvY41LBnAasAswXlXXiMhwoLANHc7BupFHqOrnfgjtWeBoYCpwMPCqiOytqp+IyNPA6cC//PYFvl6HtnOYS/2yhcAOwJv+8kNxnCJHJxDOtZ6HFwig44ZwytVHA5DZGKIxBJM+jTjjkSgPTGl7++ufb149Ykg1T8MYs4SW8zXmYW9mf5OIHHd1tqa//70stkBVG4BSbBjrIuAGVf1KVSOq+gY21PXLqCaLVfVOVW1Q1RqgASgAJohIhqouUdXo8dZGRCb5svdQ1aar/nTgVVWdoqohVX0d+5ToGX7548BhIjLAX/8FsFxV25wsSVVfVdUFquqp6pfA08D+8Z+ijlNZWemWgdJA2/dtjUFrVIYUmJTQs1stD+5Dm+Rmddl+uyknYaMhTZ9DgCGe5z2ViBDnebSmxP8eCsyKLhCRLKCfX2ck8ICI3BtVJQN7V9/EwhjZzwADgbuBMSLyDnBFlKcyALgAuEBVF0W1Gw5ojKx5wE4Aqvq9iHyBvSjuwhqbx9s7QBE5AbgUm8/JwI602CxzYxQWFrplYOfqKnIXlvH5+JhnOgLwy3GG+/YLUJiXfD271fL7N8OOv4MlpbTg+3u7bL/xEEktxwPP86bD+unYBwKrEpmKvQlnPFozB5u/OBF4J6bsl1h3byqwCPijqr68AVktfhBVDWET75NEpDc2//EYNrwFsAo4FfiXiIRU9Wl/+xKssYpmlL+9iceB80Xk38BPaOkBrccPlT0DHANMUdUGEbkDO8OmYzOxNhykd01Vq+0BDM8fEfes2I5oigth8cN2+asFUFELP90agsk9nykYtirE5nWPBzKBRmPMC8BFnueVxyvHGY8YVNUTkQuAf4rIAuCvQC1wOPAXYJKqLhCRu4HrRWQO8BU2ZrgzUKqqs9qSLSL7AeXATF9mNRCK2f+H/mivKSJSqKp/BZ4A3vFzG9Owk5odA+wT1fQFrEdzLzBVVVuF3XwKsOHKEqBRRH6CdV2/j/MUOTqBobsW8wJ5rbZnp1Y/033ZPvZeK3mkYML8PiAf+BH2JnhL7Gire7E3r3Hhch5toKpTsDmAvbChp1LgGuAyVb3Gr/MIcBv2jr8MWAxch7Xk7TEQm18oA1Zgf7Rz2tj/F8C+wDUicpWqfoT9Ue/w294GnKSqH0e1KcfmQQ6l5fuJY2V/j31+5V/AOuAq4PkNnQ9H53PfgQGqcrJabvQ8Uuwm1dEJeMa0+iSZQ4CTPc+b7Xleved5s7Gh7kMSEWI2cUp3hyNR3AXnM/7C5cza0h/j4HkMLimnYnBvqi7p/IDA5MmTAZg40U0y0EnEbQFeGPJiq2v+l8uPT5oFMcYsBPb2PG9R1LYRwPue520RrxwXtnI4ksQ2Jev4YXg/vEAAjKF3TQM7DE+2Vo7OJtVyHsDfganGmLtoDltdAjyciBBnPByOJFE5vMgaDp85w4p571AXSU43UjDncTOwHDsoaIi/fBsbCHe3hTMeDkeSqK9stFkwn1AwwIB8ZzzSjRTIcbTAf/3sYyRoLGJxV6rDkSQK19aByzmmPRHT+pMMjDETjTFtziJhjHnQGNPejBRt4oyHw5Ektiz0IOquNCOc8HNajm5ACo22uhT7jFdbPANcnogwZzwcjiRxw6StGFZin8kyEY8ztkmt8Iajc/BM60+SmOB5XptTFgEfAtskIszlPByOJNG3XxZfXd+Hp2dG2H5wkH22cPdy6UgkdXIeucaYQs/z2pqcqwDITUSYMx4ORxLpmxvg4l2d0UhnUmi01ZfAz2l73rtjgP9LRJgzHg6Hw9GFpNBoq1uAl4wxfYBXsbNcDMa+vuE67FxXceOMh8PhcHQhqWI8PM97yxhzJnAncHtU0RLgLM/z3k5EnjMeDofD0YWkUNgKz/NeAV4xxmwNFANrPM/7oSOynPFwOLohf/syzKNfhNl9eIB7DnJ/Y0didNRgROOuOoejm/HKdyF+M80DE+DzMihZV89zv8hOtlqOdkjBua06BTfMw+HoZlw2LdTi4cIXF6Zn55QupNBDgp2K8zwcjm7GyoaWnU8KPUfgaIN09Tyc8XA4uhmtuiI3PVZqkwLG3RgzKp56nufNj1emMx4ORzejuKaBZbk56zulovpGahozyMtMfiflaE2KeB5zsbcZhpa3G7Hrcb/w3eU8HI5uhgkAxjCgqp4xpVVEjOHU18PJVsvRDqmQ8/A8L+B5XtDzvABwFvACMA7I8b+fA85MRKbzPByObkZlVgbDymv56ZK1AGwTMHxoiuFn7u+cingm5e7RbwTGeJ5X66/PMcacA8wGnohXSModVSogIg+KyP1dIPcJEfl7Z8t19CzqAwEGVdWtX8+KeBSvrUmiRo4N4QVMq0+SCQAjYrZtSQIhK+jBnoeICHAtsDuQDawE3gAmqeq5MXUXAteqantz4ScdEdkHmKaqPfY37TF4Hqvzsuhd28gP/QrIjERoSHp/5GiPFByaezfwrjHmcezUJMOB0/ztcdMjPQ8RORCYAfwA7KCqRcDewBr/uyMyMztPw+QgIkZEnPFJccIGFvfO451R/VnSO4/5fQtYVpQHQGhhGev2fIjyAx4jXFLdeTtdUwkn3AnH3wmlFZ0ntydg2vgkEc/zbgdOx74E+UhgEHCG53m3JSLHeD3wNZgiMgf4QFXPaKf8CSCkqmeJyGTgcKABCAEfqepBIvIedgrjEcB+2Bkr7wCuxlrx3tgpkC9W1W+i5GYCEeAooAS4UVWf8MuHAX8HdgaygJnAb1X18yjdjvH3MQaoBR4FHgDmYZNfTT3G+ar6pIhsAdyF9bAAJgO/U9VKX54H/BY4GfsymH1V9eMETmei9LwLrpPJuKOBcMx9X2YoTMNV2ZQGr4WIf4qDhn6hmwCYPHkyABMnTmwp7K0v4djboLoeMoMQDMC5B8Pd/l8jEoGT74HnYt4hZAz87ddwzsGdfnzdhLhNwL27vN3qmr/os4NSzh1JlB7neYjIWGAr7OiCjaKqE4HFwFmqWqCqB0UVnwHcC/Tyvy8HTgEOw051/AEwVUSKotr8AngL6AucC/xNRH7qlwWAv2Ljj4OAL4B/NHk1InIo8CRwPXZSs7HAFFVdDhwKhH0dC3zDkQO8C3wHjAImAMOAe2IO80zsdMwFWIPnSHVibvoGrKsmXFnbbDgAwh6NM1dsWM6p91rDAdAYhrpG+Mt/4H/+1Ef//qy14Wja/4UufRcPKZjzwBizgzHmQmPMn4wxNzR9EpHR44wH0N//XtYJsl5R1XdV1VPVGqwrOElVZ6lqPXADEMZ6Lk18rKrPqGpIVadi59U/DUBVF6vqv1W1RlVrsTmZLbBeBsCFwIOq+h+/fYWqztiAfkcARlX/oKq1qlqGnbf/VyISnRy7Q1XnqWrY17vLqKysdMubuGxo+e5zPI8h5TVUrm07nBSPzPaora3daJ1UOCfJWo6HVBiqG40x5tfY187uB1wJ/Aj4HfamOm56Yny7xP8eCny/ibIWxqwPB9Y/oamqET/ZPnwDbRYCOwGISD9siGkfbNgr4tdpMngjgNcS0G8ksIWIrIvZ7mE9myYDGqtTl1FYWOiWN3EZz7QMmhhDaZ8Cem/Zm9LCLKhssNt7ZZG53WCik3GtZD55Udthq922BiD3+L3gzZnw1Hu0wBi476yUOSdJ+y3iINnGog2uAA7xPO8DY0yZ53lHG2MOBX6ZiJAeZzxUdbaIzAVOAKbF2SwS5/Yl2A4bABFpGhK3JKrOiJg2I4Cl/vKt2HDXrqq6QkQKgQqau4qFNHsh8ei4CJitqht7sX17x+dIQUw4AhlRQQPPWx/FKl59NbX3f4wpzCbn17tsXNjBO0LV8+2XBwLWwDx5Ebz/rQ2L7b1NSky50V1IQeMxwPO8plhkxBgT8DxvijHm2USE9Djj4fMbYLKIrALuV9XlIjIAG/tva26XlbTfaUfzBHCFiLyP7eivxJ7j16Pq/ERETgBewo7sOhY40C8rAmqAMhEpACbFyH8AeF5E/gtMBfKAH6nqh76OQREZqaoL/Pr/AW4SkauB+4AqYAjwY1VNxINxpBDhjNbR5sVFOQCYnEzyLtuza3a818buQRxtkYLGY6kxZoTneQuxDwYeZYwpxQ4KipuemPPAzzXsgU0gfy0ildgY4ABgehtNbgJOEpEyEZmyAdG3A88DbwOrsDHFg1Q1Ohj9EjahXoYdKXV+VN7ij74Oa7AjrT7C5kya9H4dO7XALcBa7FDjQ/yy2dhk+6cisk5ETvbzMPv7xzkLKAfeAXbY2DlypC5FoRBZjVHTkRhDdg8cNenoMLcB4/3lG4BnsANr/pSIkB45VNeRVNwFt4kMv7GKrJoQ84sLAMgMRyiuqWfFn9qPxbc7VNfRUeJ2J+7Y471W1/xlM/ZJGXfEGJMFZHmeV5VIu54atnI4ui212UHqI4Y9F5ZQmZ1Jr9pGcnYvTrZajnZIwbAVxphi/EcKPM+7zRjTzxjT2/O8pRtr24QzHg5HN6M6GKQuP5OqzD4U1of4tn8htSe4v3KqkgrPdURjjNkb+4iAYh8evg2b070MiNs1dVecw9HNCPqh5tqsDGqzMiASIRjokenLbkEKeh5/AY73PO8dY0yZv+0T4MeJCHFXnMPRzdhyXcvQdH59Y5I0ccRDqj0kCIzwPO8df7kpH9NAgs6EMx4ORzfj+MVLMFEDXXaev5EpSBxJJQWNx3fGmNhJyQ4Avk5EiAtbORzdjFPPGEjpnz5l6vajGbiuiosoJcGZJRybkRQwFrH8DviPMeZ1INcY8xA213FUIkKc8XA4uhlb7jOYi0vqOPjhmRRv34cf/zmhULVjM5NqxsPzvI+NMdsDvwIew86A8eNERlqBMx4OR7dk9HEjGX3cyI1XdCSdVDMeAJ7nLcOOsgLAGLOdMeZuz/OOi1eGy3k4HA5HF+KZ1p9kYIzJM8bcaIyZbIy5yxhTZIwZZYx5DTubxepE5DnPw+FwOLqQFPI8HgB2xL5P6FDsVOzjsO8IOtvzvNJEhDnj4XA4HF1IChmPg4EdPM9bbYy5D/uSu72jZthNCBe2cjh6GHUhj7llEcrr3TRjm4OIMa0+SaLA87zVAH5yvKqjhgOc5+Fw9Cg+Xh5hr+cjNPp248694dJdXDfQlXjxz6HY1WQYY/YlalLH2HXP896NW1jn6uZwOFKZn/+72XAA/G46XBrHO6McHSeFwlarsUNzm1gTs+4Bo+IV5oyHw9GDWJbQpNuOziBVjIfneSM6U57LeTgc3Y3KWpjyBcxenmxNHHGQgtOTdArO83A4uhOVtdT+6DJyF60gEgwS/seVZB4pydbK0QNxnofD0Y0oe+lzchfZiRAD4TBl1/5zk2VG3NtEu5RUeUiws3HGw+HoRqz8rOUMupFV5Zssc01NZJNlONonhYbqdirOePiIyEIROSnZemwMEZkiIlckWw/H5mfBrGqmfpdHKYNYyQgayOaHPqM3WW7EOR5dist5dBEiIsC12NchZgMrgTeASaraoRcViIgH7KmqMzpN0U1ARK4H9lDVAzZVlqoeuukaObojj962hNGrIyxmWwCWZoxi+qAd2TvJejk2TLoYi1iS6nmIyIHADOAHYAdVLQL2xo4/Tvg/ISKZnath6pDOx+aIj6ryMANLy4kAywb2oaRXH7ZcUbLJcuetc65HV5KuYatkex5/BZ5T1SubNvjexo0AIvIeME1Vb2oqj/Yq/Dv6vYAvgJOBL0RkiF/1bRGJAC+o6lkiUgzcDRyIfaLyLeASVV0bpc8oEZkB7ADMAs5T1c/8/e4P3AKMBULAO8BFqrraL88ELgdOBYZgH8i5AnuOrwYCItI0yn47VZ0vInsCtwITgDL/fNylqp6I7ANMA04H/gT0Bwqjz4mIjAAWAMNVdamvx2nAtaq6lb++EPg7sD+wi1//V8A2/nnuD7wMnKuqoQ39WI7kklNXz9q++ZQNykRKlMbMDCrD29Ewfy1Zt74As5bB2QfAKfvCglVw8aNQXQ83n8i8uiKeLNm6Tbn/nAs/HbaZD6YHkS4J8liSZjxEZCz29WfnbaKovYDXgeFAhqrW+AbmoJiw1bNAI7ajBngGeBo4PKrOudg3an0NXAq8ISKjVbUCqAcuAL4E+gEvAfcAJ/htb/LbHue3Hwr0VdWZIjKemLCViGyDDc+dBPwHGANMAUqAp/xqQezslzv6uneUU4EjgbnA48BrWOO3PVAMKPAu8Nwm7MPRhXiex0++nMUHu0xg3Op5ZC+vZ+uyeRT1rqTu92vJesl/JfVHP8DOo+HXf7PLAEfcwk1X38eaSF6bsqsaNtNB9FBSaHqSTiWZYav+/veyTZSzWFXvVNUGVa1pq4LvjRwMXKqqZapahjUOh4nI4Kiqj6rq56raAEwCaoEjAFR1hqp+pqohVV2JfZHK/r58A5wPXK6qM1XVU9WlqjpzA3qfB7ysqv9S1bCqzgLuB06JqXeVqpa3d2xx8rCqfq+qjVgDMQq4RlWrVXUx8B7WK+lyKisr3XIHl2ePHko4I5Nvh4zjwd1PoTYjm151lbCmYn09IhFYuY7IsjXN29ZUUlff/l89GEidY+xuy/HgwladT1Owdijw/SbIWRhHneH+94KobfOiypoS8+tl+aGjxcAwABHZGRu22h7Iw4a+Cvzq/YF8YHYCeo8E9hORY6K2BbCvhGwiErPeUaIHHtQAYVUtidlW2An72SiFhYVuuQPLRYWFrO7ba/16XVYuFdkFzM8fw6gLDqNIv4XyGth3W9hrAoHrfgG/ftAak8t/xsRBS3hhzRjawoukxjF2x+V4SNeEedKMh6rOFpG52LDPtHaqVWE7ZWC9BxFLW4PUYzOATR3wCGzoBponAIvunEdE7csAWwBN7/V9AXgFOE5VK0TkCGCyX1YCVGNDT3Pi1HER8Jiqnt9G2frjUNUNZTObcij5UdvaOkeO7k7TEE/PA2MYtrKEVdXjIZJF/yO3gYUPwYoyGDsYgkE48wA4dCeoa4BRgzhx8mT2K1rGrxfs00r01sWb/3B6Es54dA2/ASaLyCrgflVdLiIDgDOB+dhY/C9E5C6gDrg5TrkrsR35DABf7tvAnSJyKtZruBOYEjMc+AwReQ2bs7gE62G87pcVAeVApYhsAVzV1Mj3Uv4G3OZ7K99iO/G+qvq1r88WIpLlh8TAJseni8ibwJtYgzcW6K+q0+M5SFUtFZFFvt5XY/M5ZwPhOM+TozuRncHQ5SvYc9aXFK0JU0A5A2urCQQM9M63n2iG9G2xOiirtk2xh7pXoXcpkfS0HckdqquqU4E9sJ3e1yJSCXwIDACmY0dHzcKGmP6P5o58Y1wD3CAiZSLykL/tJKDSlzcLWEfr/MLDwL38f3t3Hm/XeO9x/PM4iUjkEFPMkSCoUi0/M0W9OqCpuanrihaJS7VCXVddU+lFqdY8RqmhxpqiLdfcmv3MSk0hyCAJEYnkJjnJc/941pGdfU6Svc85ezhrf9+v13qdNT77t3d21m8/w1orjXwaCuzh7q2X8I4ADsvKuIM0Qqn4NW8F7sr2eYyUwMj2/RCYaGafmdkgd3+N1J8yktSsNAm4lgV9QaU6OCtnGvA74Ooyj5duYvvvrsg60z5kmylPsXF8lgG8yewVO//7r2dTTs9udSKvFwmGqPvaSHXpC9cJjw+9je1vvenL8TvjVh/AmuPPX+Jxo0enFtYfvNn2GtOJRwRWXbapK8NsBCVngCOGvtnmO3/ZLRt2+wyi25OIdCPr77bOQmetZdZbudNl9u+j00Al5bXmoW+NSDey2oGbM2OrrwIwt3lZVjr/gCUcsWQhJyezejU/tJ3yoNYd5iJSjp496PvUr2DMx/RctR809651RLIEebmuo5iSh0h3s9RSsP7qS95PpILUbCXSQNZZrtYRNB71eYhIt/fID5voUXDu+nq5A8OlbOrzEJFub1C/wGc/a+KKV9Lt+g7fLCdnsjqW1xsjKnmINJhllw4ca/k8odUjdZiLiEjZlDxERKRseenjKKbkISJSQfNz2ueh0VYiOfXWp/O5/c35fD67vScCSLXkrFuIegAAE05JREFUdaiuah4iOXTR8y38/JE0H4Ab1utBcw89or4W8tpspZqHSA4d/ciC+Qic+uHmNYul0ekxtCLSbRTfA/ydOf1qEofkt89DyUOkIeTzBNYdzMvpR6/kISJSQXlppiqm5CEiUkF57TBX8hARqSD1eYiISNnm5bTZSkN1q8jMopntUOs4REQ6q9vXPMzsUWBbYA4wH/gEeAI4392fr2FoVWVm1wIt7n5YrWMRkQXy2ueRl5rHGe7e7O7LA7sAY4GnzWzvjhRmZj27NLpuolHfd0N672MI+yyYeu4H8+bVOqpcmkdoM+VBt695FHP3scBJZrY6cJGZ3QX0Bk4H9gWWB54FjnL3d+DL2stLwEDgW8CZZrYR0ATMBfYBvgCOA94ArgI2Ahw40N3HZ+UcDRwBrAlMBW4ETnL3wv+VZmYXA+tlxw8viONHwC+BQdnr3QMc6+5fZNv7Aqdl8awCfAAcDmwHHFhQBsDy7j7PzPYCTs5ebwLwa3e/Mdv3x8BJwBXA0cA04Ktlf+jSDRSdsNY9YuHllvmw3hHw/pXVC6lB5PU6j7zUPNpzM+kkviEwinSy3wZYDXgGuLfol/YhwIWk5HJhtm4/4M/AisAZpKRxOrA3sCrpQt7TCsr4CNgNWA7YMyuzuBlpRFZuf+CfwD1m1pRtmwb8G9AP2DGbTio49mpga2DX7DX2Aia6+zmkRPVHd++bTfPM7NvZMSOz93AwcLGZfbOgzIHAGsBgYMv2PsiuNH36dM1XYb7tNebt7VNk3Kd1E393mS+Fbk/S/XyU/V0VOABYx90/BjCzX5FOqFsDj2f73e7uD2fzM80M4GF3/0t2zHXAZcD17v5Rtu52YHjrC7r7nwte/0Uzu550or+iYP15BTWN40k1lK2BJ939bwX7vWNmlwLDsn37Az8ENnH397J93l7CZ3A0cIG7/yNbftbMbsjK/Hu2bi5wgrvPXkJZXaK5uVnzVZhve0V5XHif5XrD57MW3uXgnesm/u4yX4q8jrbKc/JYK/vbej/qV7KE0KonsHbB8vvtlDGhdcbdWxPKhILtM4Evv0lmdgBwLLAu6bNdGni6qMwvXycrc3JrrFlN4RRSLakXqdlsUrb7wOzvW+3EuSiDgF3M7NiCdU3APwqWJ1QrcUgtFZ3Apt0Iax4K46em5SO/C5ccXv2wGkBe72Wc5+QxFBjHgpPtYHefvJj9O/XQAzNbG7iB1B/xN3efY2a/Baxo14EFx/Qh9V18ZGZLA3cBxwN/cPdZZnYUqZ8FFiSdwcDrJcY/FrjW3c9dTOh62ENDaKcZa9zV1Q+jAanm0U1kJ/HDgB8DQ939YzP7E3CpmY1093Fm1o80KusBd5/RRS/dl9SHNBmYa2bbAAeROtgLHZN10I8DzgbGkPpgegPLAFOzxLExcFTrQe4+KWsmuzTr6B5L6gQnawabCGxjZku5e2tCOB+4xsyeBp4k1To2BYK7exe9bxFZjJZ85o7cdJifbGbTzexzUlv++sB2BX0Qw4E3gUfNbDrwKrA/i+pV7AB3fwM4Fbgb+Aw4AbipnV1HAXeQksxmwJ7uPi9LYkcA55jZDOAS4E9Fxx5CGhX2GDA9e63VCspdFvjEzD4zsyZ3/19SB/25wBRSk9vvSYlORKqghdBmyoMQY5edP0VKoS9cFYTfFre0z+eeDe9nyJAhNYknh0rOAOv/bHKb7/w7F63S7TNI7pqtRETqyVz1eYiISLnm1jqAClHyEBGpoJmqeYhId9UD3beqVmblM3coeYjkUc8Acwu6abfoM6V2wTS4OTkZXVUsL0N1RaTA4weEL09Z/XrBCWu9VNN4GlpoZ8oB1TxEcmirNZpo+UVk+hxYvldg9OhaRyR5o+QhklNLhcDyvWodhZDTDnM1W4mISNlU8xARqaSc1jyUPERybNbcyGyN0q2tfOYONVuJ5NUxD7fQ54J5rHDxPEa+t02tw2lg+RxupZqHSE6d/8KC+TFzVmDS7KVrF0wjy0euaEPJQ6RBvDh9xVqH0JiUPESkO3t5Rr9ah9Cg8pk9lDxEGsS7s5trHUJjymfuUPIQaRRR42NqJJ/ZQ8lDpEHMzelJrO7l9GNX8hARqaScJg/VY3PIzKKZ7ZDN72xmxQ+0lgYUmF/rEBqUrvPIDTN7FNgJGOrutxas3xp4Ghjr7gNrE51IZeiXYo3kI1e00cjfpzeA4UXrhmfrO8TMenYqIpEKmt7eb8VjroawT5o2PLL6QTWCENpOOdCQNY/MHcB/mNm67j7GzJqBfYEzgZ8CmFkf4CxgH6A38Djwc3f/INv+KPASMBD4FnCmmZ0HnAMcCMwHfgeMAH7t7tea2VrAKGALYGngFWCkuz+flXkasCPwDHBYFutl7n5qa+BmtglwXlbGTOBG4BR3n7ukN21mu2bvcQOgBXgoe0+TyvjspBuaRe+FV4y4FK56cMHyWxNhtZ/AxGuqG5h0S41c8/g/0kn30Gz5AOAxYELBPr8HtsmmdYApwGgzayrY5xDgQmD57O8vgd2yYwYBa2XHtloKuDRbtxrwAnBHUa3lm8AHwBrAEOBEM9sewMz6Z3HekW3fFvh29rqlmA0cBawCbJqVcUGJx0q3VvTf/eYn2u7y8bTqhCLdXiMnD4CrgJ+YWQ9S7eCq1g1mthQwDDjJ3ce5+xfASOArwFYFZdzu7g+7e3T3mdkx57j7GHefBfwXLOipdPcP3P0ed5+ZbT8JGAAMLijzLXe/3N1b3P0ZUu3Gsm3DgJfd/Qp3n+Pu40i1o2GlvGF3f9zdn8vKnkiqJe1a4ufVadOnT9d8lecLFa6fu8labXfovXRdxNxd5kuSz/7yhm62wt1fM7OxwMnAqsB9pBoIpF/mywBjCvafYWaTgLWBp7LV7xcVuyYwtuCYWWY2uXXZzFYmNWXtDPRjQWJZpaCMwtoPwBdA6+XBg4Dtzeyzgu0BaKIEZrYFqdlqM6BPdmzfUo7tCs3NzZqv2vzCg+z68gXNzSt9udzzibNh8JHw7sdpRa8e8OkfaV6mV1E5ml/UfGlyki2KNHTyyFwJXA2c7u7zzFp/4DOZ1MQzCHgXwMz6Av2BDwuOLx7/OI6CZioz683CieEsYHVga3efkPW1fE7p37CxwIPuvkeJ+xe7Gbgd2N/dPzez7wN6wnUD6EnRgz1CgHcuq00wjSSfuUPJA7iJlAyeL1zp7vPN7DrgDDN7HfiM1En9L+DZxZR3PfCfZvYIqQZxFgs3Dy5H6uSemiWj35QZ73XAL8zsEOBPwBxSh/0G7n5fCccvB0wDppvZAOCEMl9fuq1Gb6WukZwmj4b/Nrn7/7n7g+4+tZ3NxwAOPEfqwF4d+IG7L+7ZbGcBD5ASzPukBDKeVIsBOJVUe/mENNLqSSj+SbjYeCcCuwB7ZeVPBe4E1i2xiBGkUVzTSZ3ut5X62iLSEfns9AgxxlrHkGtZ7WIqsJO7P1nreOqAvnBVEn67cJ/HynzC5ONWrVE0uVNyBginzmrznY+/6t3tM4iarbqYma0AbE26fqIPabjvWFLtRaSGuv35SupIwzdbVUAT8GvgU+A90nUeQ0q5gE+kkpZW7qiNfLZaqebR1dx9CguuyRCpG5v2mULqbpOqysntSIqp5iHSIAb2+qLWIUiOqOYh0iA26lPmldHSNfJZ8VDNQySvNi8YWLUMc1hvWdU8aiOfnR6qeYjk1PMH9eDhsfOZMgt6vfVQrcNpXPnIFW0oeYjk2LfWSY0Lo9+ucSCSO0oeIiKVlNOah/o8RESkbKp5iIhUkq7zEBERSVTzEBGppHxWPFTzEBGR8qnmISJSSTmteSh5iIhUVD6zh5KHiEgl5TN3qM9DRETKp5qHiEglqeYhIiKSKHmIiFRSiXdkDyG8H0LYpHqBdY6Sh4iIlE3JQ0SkkkJoO5V8aBgWQng1hPBKCOHOEEL/bP1TIYQts/lLQwj/zOZ7hBCmhBCWrch7KaDkISJSSR18kGDWhHU28J0Y49eA14CLss0PAbtm8zsAs0IIqwNbAm/EGCv+2EiNtpKqCiHcD6zc0eN79OixcktLy5QuDKlL1XN8iq3j2onvvhjj90o5Nh7Xo6PjrXYB/hpjnJAtXwG8nM0/DJwYQrgR+AR4jJRMBpESS8UpeUhVlfofblHMzN3duiqerlbP8Sm2jqtRfAGIRetal58ANgf2ICWLx4BDSMnjlGoEp2YrEZH69BCwewhhtWx5OPAgQIxxNvACcEK27mlge+Br2XzFqeYhIlI/HgwhtBQsnwg8EEKIwBjg8IJtD5H6ODzG2BJCeAd4L8Y4pxqBKnlId3NlrQNYgnqOT7F1XMXjizEOXMSmPy5i/7OAswqWd69AWIsUYixuUhMREVk89XmIiEjZ1Gwldc3M+gDXAFsALcBx7n5vO/vtSRpl0os0SuUP7n5eHcW3JnADaYTM25UauWNmG5CaOVYiDeEc5u5vF+3TBFwIfI80eudsdx9ViXg6ENt3gDOBTYGL3P24SsdVZnwnAz8i/Vu3ACe6+/3VirGeqOYh9e44YLq7rw8MAUaZWd929psIDHH3TYDtgCPMbMc6im8GcCpwYIXjuRy4xN03AC4hXRtQ7EBgfWAwsC1wmpkNrHBcpcY2hjSq6NwqxFOslPieBbZ0981IQ2NvMbPeVYyxbih5SL0bSvpPTfYr0IHdindy92fcfXw2Pw14A1injuKb5u5/JyWRijCz/qSazU3ZqpuAzc1slXZivsrd57v7ZOAuYP9KxVVObO7+jru/SPpVXzVlxHe/u8/MFl8h1XJXqlqgdUTJQ+rdAGBswfIHwNqLO8DMNgK2IV2FW2llx1dBawPj3H0eQPZ3fDvx1CLmUmOrlY7ENwx4190/qkJ8dUd9HlJTZvYC6WTWnlU7UN7qwN3AT1trIp3R1fFJPpjZTsAZwLdrHUutKHlITbn75ovbbmYfkJqfJmerBgCPLGLf/qSrbc9191vrLb4q+BBY08ya3H1e1jG+Rra+UGvMz2XLxTWRWsZWKyXHZ2bbkgY/7Onub1Y5zrqhZiupd7eRXVVrZoNJV9TeV7yTma0EPABcXI2RQ+XGVw3uPgl4CTggW3UA8GLWr1HoNmC4mS2VtenvBfy5TmKriVLjM7MtgVuA/dz9hepGWV90kaDUNTNbFrgW+AYwDzje3e/Otp0OjHf3y83sXOAooPCX4AXufk2dxNdE+nXfC1gemASMcvfTujiejUjDTVcAppKGm75pZn8FTnF3z2K5GPhOdthv3L3iV1CXGNsOwM3AcqTO6GnAodUYDltifM8BA4FxBYce5O6vVjq+eqPkISIiZVOzlYiIlE3JQ0REyqbkISIiZVPyEBGRsil5iIhI2ZQ8RGSRQggxhLBDreOQ+qPkIdJJIYR1Qwi3hRAmhhBmhBA+DCHcGUJYOtv+4+wRocXHLWr9v2cn7VPa2fZoCGF29jrTQggvhhD2XURcd4cQrlvEtkdCCBeX/25FEiUPkc77KzAB2BBoJt3m/H7SRW4dMQL4FDgshNDUzvYzYox9SXdzvQm4JYSwQTv7XQHsF0LoV7gyhDAY2In6f/Sr1DElD5FOCCGsREoal8cYp8Xkoxjj5THG2R0o7yvAjsDBwOq0c3v3VjHGFuBSoIn08KRi95HuuXVQ0foRwNMxxldCCGeGEMZkNZl3QwgjFxNbm5pSCOHaEMKoguUBIYTbQwgTsunKEELz4t+1dEdKHiKdEGP8BPgnMCqEMCyEsHEIoaM1Dkj3yXo1xngvqUYzYlE7Zs1iPwXmAi+3E9t8YBTp4UqFxxzMglrH68AOpBrTcOCsEMJ3OxJ4CGEZ0m3wXwfWBTYG1gIu6Eh5Ut+UPEQ6b2fgUWAk6eZ6H4cQTi5KIoNCCJ8VTqRaw5eyk+9BwB+yVVcDu4cQ1ip6vf/Ojv8I2BPYN8bYpu+koIyvhBC2zpb3BnoCtwLEGG+IMY7PakwPA38Bdi3z/bf6PumWR6fEGGfFGKcCJwMHLqL5TboxJQ+RTooxTokxnhhj3BzoBxxPep76Twp2ey/G2K9wAo4sKmp/oC/pdt+Qah6TgMOK9vufrIz+McbtYoyjFxPbeFJCaK3BjACujzHOBAgh/DyE8GoIYWqWkIYAxU8eLNUgYEBRgnyI9Jz01TpYptQpJQ+RLhRjnBljvJb0iNKvl3n44aT+i9dCCBNJNYsVgUM7+cv9CmBoCOEbwC5kTVYhhO2B32Svu3KW0Eaz6I7+GcCyRevWKJgfC7xVnCRjjMvEGMchuaLkIdIJIYQVQghnhRA2CSH0DCH0yIbObgL8o4xyNga2JzUrfb1g2or0q333ToR5PzCF9MyOp2KMr2XrlyPdRn4yEEMIe7CYDnrgRaB/COH7IYSlQgh7A98s2H4v0DOEcGIIoTkka2b7Sc4oeYh0zhygP3AHaXjtZOAk4GcxxtvKKOdw4IUY4+gY48SC6RUKHjjVEVnH+VWkZqXC4bn3A9cDz5KSy37AnYsp513g6KyMT4HvUfAQqawpbFdSR/m/SM/ieIjya2DSDeh5HiIiUjbVPEREpGxKHiIiUjYlDxERKZuSh4iIlE3JQ0REyqbkISIiZVPyEBGRsil5iIhI2ZQ8RESkbP8PuKHtmi21ZqwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_CR, holdout_X_raw_CR, show=False, plot_size=(4, 5), color_bar_label='Read Count', max_display=other_X_raw_CR.shape[1])\n",
    "fig, ax = plt.gcf(), plt.gca()\n",
    "ax.set_xlabel('SHAP Value')\n",
    "plt.savefig(results / f'hold_{hold_out}_raw_CR_shap.png', dpi=600, format='png', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Septic</th>\n",
       "      <th>Healthy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Train fold</th>\n",
       "      <td>102</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Validation fold</th>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Test fold</th>\n",
       "      <td>117</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Septic  Healthy\n",
       "Train fold          102       35\n",
       "Validation fold      11        3\n",
       "Test fold           117      170"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_precision</th>\n",
       "      <th>test_recall</th>\n",
       "      <th>test_F1</th>\n",
       "      <th>test_AUROC</th>\n",
       "      <th>test_AUPRC</th>\n",
       "      <th>external_test_precision</th>\n",
       "      <th>external_test_recall</th>\n",
       "      <th>external_test_F1</th>\n",
       "      <th>external_test_AUROC</th>\n",
       "      <th>external_test_AUPRC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Raw</th>\n",
       "      <td>0.982</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.404</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.569</td>\n",
       "      <td>0.501</td>\n",
       "      <td>0.431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RA</th>\n",
       "      <td>0.974</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.995</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.416</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.575</td>\n",
       "      <td>0.636</td>\n",
       "      <td>0.528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Raw CR</th>\n",
       "      <td>0.970</td>\n",
       "      <td>0.859</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.941</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.026</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.457</td>\n",
       "      <td>0.420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RA CR</th>\n",
       "      <td>0.917</td>\n",
       "      <td>0.876</td>\n",
       "      <td>0.889</td>\n",
       "      <td>0.883</td>\n",
       "      <td>0.894</td>\n",
       "      <td>0.441</td>\n",
       "      <td>0.419</td>\n",
       "      <td>0.430</td>\n",
       "      <td>0.535</td>\n",
       "      <td>0.504</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        test_precision  test_recall  test_F1  test_AUROC  test_AUPRC  \\\n",
       "Raw              0.982        0.948    0.963       0.988       0.970   \n",
       "RA               0.974        0.964    0.968       0.995       0.967   \n",
       "Raw CR           0.970        0.859    0.909       0.930       0.941   \n",
       "RA CR            0.917        0.876    0.889       0.883       0.894   \n",
       "\n",
       "        external_test_precision  external_test_recall  external_test_F1  \\\n",
       "Raw                       0.404                 0.957             0.569   \n",
       "RA                        0.416                 0.932             0.575   \n",
       "Raw CR                    1.000                 0.026             0.050   \n",
       "RA CR                     0.441                 0.419             0.430   \n",
       "\n",
       "        external_test_AUROC  external_test_AUPRC  \n",
       "Raw                   0.501                0.431  \n",
       "RA                    0.636                0.528  \n",
       "Raw CR                0.457                0.420  \n",
       "RA CR                 0.535                0.504  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(split_sizes)\n",
    "display(final_results)\n",
    "final_results.to_csv(results / f'hold_{hold_out}_out_model_results.csv', index=True, header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xgboost",
   "language": "python",
   "name": "xgboost"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}