Export - Computation Time.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "94860052-5b6c-42f7-b88b-917d216a0438",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python  3.6.13\n",
      "Python Executable: /home/ubuntu/anaconda3/envs/cfshap22/bin/python\n",
      "CWD =  /home/ubuntu/air/xai/cfshap-release/cf-shap-facct22\n",
      "NumPy 1.19.2 | Pandas 1.1.5 | SciPy 1.5.2 | NetworkX 2.5.1 | StatsModels 0.12.2\n",
      "scikit-learn 0.24.2 | xgboost 1.3.3 \n",
      "MatPlotLib 3.3.4 | Seaborn 0.11.1 | "
     ]
    },
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-2.2.0.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from emutils.imports import *\n",
    "from emutils.utils import load_pickle\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c48d808b-607a-488f-91c3-8e50597792d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from constants import DATA_DIR, MODEL_DIR, RESULTS_DIR\n",
    "\n",
    "parser = ArgumentParser(sys.argv)\n",
    "\n",
    "# General\n",
    "parser.add_argument('--data_path', type=str, default=DATA_DIR, required=False)\n",
    "parser.add_argument('--results_path', type=str, default=RESULTS_DIR, required=False)\n",
    "parser.add_argument('--model_path', type=str, default=MODEL_DIR)\n",
    "\n",
    "args, unknown = parser.parse_known_args()\n",
    "args = attrdict(vars(args))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86100628-0429-46fb-8e04-61faf4a09f19",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36890d3c-4332-4a90-bce3-9390c7779055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b41a2d625cce4e28824b7409d68d7dd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-09-27 12:17:34] INFO:\t Loading PICKLE from ./results/heloc_Dv2Mv5_xgb_profiling.pkl...\n",
      "[2022-09-27 12:17:34] INFO:\t Load done in 0.0007 seconds.\n",
      "[2022-09-27 12:17:34] INFO:\t Loading PICKLE from ./results/lendingclub_Dv2Mv5_xgb_profiling.pkl...\n",
      "[2022-09-27 12:17:34] INFO:\t Load done in 0.0005 seconds.\n",
      "[2022-09-27 12:17:34] INFO:\t Loading PICKLE from ./results/wines_Dv2Mv5_xgb_profiling.pkl...\n",
      "[2022-09-27 12:17:34] INFO:\t Load done in 0.0006 seconds.\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for dataset, data_version, model_version in tqdm([\n",
    "        ('heloc', 'v2', 'v5'), \n",
    "        ('lendingclub', 'v2', 'v5'), \n",
    "        ('wines', 'v2', 'v5'), \n",
    "    ]):\n",
    "    results.extend(load_pickle(profiling_filename(args, dataset, data_version, model_version, model_type = 'xgb')))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2b42402-8224-4d97-80cb-f76f32244213",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b0ba2ac-0725-4cf2-bfd9-4b1c530f2329",
   "metadata": {},
   "source": [
    "Let's check the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9e30ea9f-5ce8-4a0e-a03c-f0159af81443",
   "metadata": {},
   "outputs": [],
   "source": [
    "from emutils.utils import pandas_max\n",
    "pandas_max(200, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "554ce247-9684-4109-931a-9b49213ac4de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Method</th>\n",
       "      <th>CF-SHAP $10$-NNQ+L1</th>\n",
       "      <th>CF-SHAP $100$-NNQ+L1</th>\n",
       "      <th>SHAP D-LAB (n = 100)</th>\n",
       "      <th>SHAP D-PRED (n = 100)</th>\n",
       "      <th>SHAP TRAIN (n = 100)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HELOC</th>\n",
       "      <td>$202 \\mu s$</td>\n",
       "      <td>$764 \\mu s$</td>\n",
       "      <td>$668 \\mu s$</td>\n",
       "      <td>$645 \\mu s$</td>\n",
       "      <td>$422 \\mu s$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LC</th>\n",
       "      <td>$579 \\mu s$</td>\n",
       "      <td>$3,739 \\mu s$</td>\n",
       "      <td>$3,914 \\mu s$</td>\n",
       "      <td>$3,932 \\mu s$</td>\n",
       "      <td>$2,215 \\mu s$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WQ</th>\n",
       "      <td>$244 \\mu s$</td>\n",
       "      <td>$930 \\mu s$</td>\n",
       "      <td>$876 \\mu s$</td>\n",
       "      <td>$872 \\mu s$</td>\n",
       "      <td>$392 \\mu s$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Method  CF-SHAP $10$-NNQ+L1 CF-SHAP $100$-NNQ+L1 SHAP D-LAB (n = 100)  \\\n",
       "Dataset                                                                 \n",
       "HELOC           $202 \\mu s$          $764 \\mu s$          $668 \\mu s$   \n",
       "LC              $579 \\mu s$        $3,739 \\mu s$        $3,914 \\mu s$   \n",
       "WQ              $244 \\mu s$          $930 \\mu s$          $876 \\mu s$   \n",
       "\n",
       "Method  SHAP D-PRED (n = 100) SHAP TRAIN (n = 100)  \n",
       "Dataset                                             \n",
       "HELOC             $645 \\mu s$          $422 \\mu s$  \n",
       "LC              $3,932 \\mu s$        $2,215 \\mu s$  \n",
       "WQ                $872 \\mu s$          $392 \\mu s$  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def condition(m):\n",
    "    if m in ['training_100', 'diff_pred_100', 'diff_label_100', 'knn100_qL1', 'knn10_qL1']:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "results_ = [r for r in results if condition(r['Method'])]\n",
    "\n",
    "df = pd.DataFrame(results_)\n",
    "del df['Runtimes'], df['Model']\n",
    "for col in ['Mean','Minimum', 'Maximum','Variance']:\n",
    "    df[col] = df[col] .apply(lambda x : f\"${x * 1e6:,.0f} \\mu s$\")\n",
    "\n",
    "df = df[['Method', 'Dataset', 'Minimum']]\n",
    "df = df.sort_values(['Dataset'])\n",
    "df['Method'] = df['Method'].apply(method_to_name)\n",
    "df = df[df['Dataset'] != 'Toy Moons']\n",
    "\n",
    "\n",
    "df = df.pivot(index = 'Method', columns = 'Dataset')\n",
    "df.columns = df.columns.droplevel()\n",
    "\n",
    "map_cols = {\n",
    "    'GMSC': 'GMSC',\n",
    "    'HELOC': 'HELOC',\n",
    "    'Lending': 'LC',\n",
    "    'Wine': 'WQ',\n",
    "}\n",
    "\n",
    "def map_colsf(c):\n",
    "    for a, b in map_cols.items():\n",
    "        if a in c:\n",
    "            return b\n",
    "\n",
    "df = df.rename(columns = {col: map_colsf(col) for col in df})\n",
    "\n",
    "df.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "357fbae7-53ca-4f3c-836b-d190e331221e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{rllll}\n",
      "\\hline\n",
      " &        HELOC &             LC &           WQ \\\\\n",
      "                &              &                &              \\\\\n",
      "\\hline\n",
      "CF-SHAP $10$-NNQ+L1   &  $202 \\mu s$ &    $579 \\mu s$ &  $244 \\mu s$ \\\\\n",
      "CF-SHAP $100$-NNQ+L1  &  $764 \\mu s$ &  $3,739 \\mu s$ &  $930 \\mu s$ \\\\\n",
      "SHAP D-LAB (n = 100)  &  $668 \\mu s$ &  $3,914 \\mu s$ &  $876 \\mu s$ \\\\\n",
      "SHAP D-PRED (n = 100) &  $645 \\mu s$ &  $3,932 \\mu s$ &  $872 \\mu s$ \\\\\n",
      "SHAP TRAIN (n = 100)  &  $422 \\mu s$ &  $2,215 \\mu s$ &  $392 \\mu s$ \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df.to_latex(bold_rows=False, escape = False, column_format='rllll')\n",
    "      .replace('toprule', 'hline')\n",
    "      .replace('midrule', 'hline')\n",
    "      .replace('\\\\bottomrule\\n', '')\n",
    "      .replace('Dataset', '')\n",
    "      .replace('Method', '')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbd99a55-01d6-4f77-9cea-b68a672c07c0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}