2a-Baselines-Occutherm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Notebooks\n",
    "import nbimporter\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# Functions from src\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "# Defined Functions\n",
    "from utils import *\n",
    "\n",
    "# Pandas, matplotlib, pickle, seaborn\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from statistics import mean \n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import SMOTE, SMOTENC, ADASYN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global constants\n",
    "num_trials = 30\n",
    "test_size_percentage = 0.2\n",
    "fixed_depth = 10\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load TCS Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Temperature (Fahrenheit)  SkinTemperature  ClothingInsulation  \\\n",
      "count               1508.000000      1508.000000         1508.000000   \n",
      "mean                  71.453707        85.207610            0.558176   \n",
      "std                    6.221285         5.362427            0.198067   \n",
      "min                   60.070000        62.986781            0.000000   \n",
      "25%                   65.599998        81.927500            0.410000   \n",
      "50%                   70.199997        85.376000            0.490000   \n",
      "75%                   77.634998        88.598001            0.680000   \n",
      "max                   85.000000       110.235782            1.070000   \n",
      "\n",
      "        Height(cm)  Shoulder Circumference(cm)  Weight(lbs)       Gender  \\\n",
      "count  1508.000000                 1508.000000  1508.000000  1508.000000   \n",
      "mean    169.909218                  109.055637   152.835411     0.443634   \n",
      "std       9.215815                   10.985466    30.818397     0.496978   \n",
      "min     151.000000                   89.500000    90.000000     0.000000   \n",
      "25%     163.300000                  101.600000   126.000000     0.000000   \n",
      "50%     170.000000                  106.900000   146.000000     0.000000   \n",
      "75%     176.700000                  117.000000   173.000000     1.000000   \n",
      "max     189.000000                  132.000000   236.600000     1.000000   \n",
      "\n",
      "       Temperature_outside  Humidity_outside  Discrete Thermal Comfort_TA  \n",
      "count          1508.000000       1508.000000                  1508.000000  \n",
      "mean             49.839702         70.359284                    -0.257294  \n",
      "std              20.873157         13.296121                     0.906428  \n",
      "min              10.510000         33.500000                    -2.000000  \n",
      "25%              35.540001         62.000000                    -1.000000  \n",
      "50%              47.240002         69.199997                     0.000000  \n",
      "75%              69.260002         79.400002                     0.000000  \n",
      "max              91.400002        100.000000                     2.000000  \n"
     ]
    }
   ],
   "source": [
    "df_tcs = pd.read_pickle(\"data/occutherm/df_feature1.pkl\") \n",
    "del df_tcs['Participant_No'] # original dataset contains participant number\n",
    "\n",
    "df_tcs_train = pd.read_pickle(\"data/occutherm/df_feature1_train.pkl\") \n",
    "df_tcs_test = pd.read_pickle(\"data/occutherm/df_feature1_test.pkl\")\n",
    "\n",
    "dataset_string = \"occutherm\"\n",
    "\n",
    "# total count for instances per class: 818\n",
    "print(df_tcs_train.describe())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification models on train data (imbalanced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features: ['Temperature (Fahrenheit)' 'SkinTemperature' 'ClothingInsulation'\n",
      " 'Height(cm)' 'Shoulder Circumference(cm)' 'Weight(lbs)' 'Gender'\n",
      " 'Temperature_outside' 'Humidity_outside']\n",
      "Expected accuracy (f1 micro) based on Cross-Validation:  0.4958677685950413\n",
      "GaussianNB(priors=None, var_smoothing=1e-09)\n",
      "Number of folds: 10\n",
      "Best parameters set found on development set:\n",
      "{'algorithm': 'brute', 'metric': 'seuclidean', 'n_neighbors': 14, 'weights': 'distance'}\n",
      "Number of folds: 10\n",
      "Best parameters set found on development set:\n",
      "{'C': 1000, 'class_weight': 'balanced', 'gamma': 0.1, 'kernel': 'rbf'}\n",
      "Number of folds: 10\n",
      "Best parameters set found on development set:\n",
      "{'class_weight': 'balanced', 'criterion': 'gini', 'min_samples_split': 2, 'n_estimators': 100}\n"
     ]
    }
   ],
   "source": [
    "acc_rdf, rdf_real_model = train_rdf(df_tcs_train, rdf_depth=fixed_depth, test_size_percentage=test_size_percentage)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nb acc CV: 0.5463576158940397\n",
      "knn acc CV: 0.6821192052980133\n",
      "svm acc CV: 0.6688741721854304\n",
      "rdf acc CV: 0.704251655629139\n"
     ]
    }
   ],
   "source": [
    "print(\"rdf acc CV: {}\".format(acc_rdf))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_pickle(rdf_real_model, \"models/\" + dataset_string + \"_rdf_reall_full.pkl\")\n",
    "save_pickle(acc_rdf, \"metrics/\" + dataset_string + \"_rdf_reall_full_acc.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variability baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "Thermal Comfort: 0\n",
      "Thermal Comfort: -2\n",
      "Thermal Comfort: 1\n",
      "Thermal Comfort: -1\n",
      "Thermal Comfort: 2\n",
      "52.50489290173811\n"
     ]
    }
   ],
   "source": [
    "variability_baseline_list = []\n",
    "\n",
    "for _ in range(0, num_trials):    \n",
    "    variability_baseline = evaluation_variability(df_tcs_train)\n",
    "    variability_baseline_list.append(variability_baseline)\n",
    "\n",
    "mean_var_baseline = mean(variability_baseline_list)\n",
    "print(mean_var_baseline)\n",
    "save_pickle(mean_var_baseline, \"metrics/\" + dataset_string + \"_variability_baseline.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "# Diversity baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8966552371208951\n"
     ]
    }
   ],
   "source": [
    "diversity_baseline_list = []\n",
    "\n",
    "for _ in range(0, num_trials):\n",
    "    diversity_baseline = evaluation_diversity(df_tcs_train, df_tcs_train, baseline=True)\n",
    "    diversity_baseline_list.append(diversity_baseline)\n",
    "\n",
    "mean_diversity_baseline = mean(diversity_baseline_list)\n",
    "print(mean_diversity_baseline)\n",
    "save_pickle(mean_diversity_baseline, \"metrics/\" + dataset_string + \"_diversity_baseline.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quality of the final classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_acc_test, class_acc_train, class_models, class_report_rdf = evaluation_classification(df_tcs_train, \n",
    "                                                                                                df_tcs_test, \n",
    "                                                                                                rdf_depth=fixed_depth, \n",
    "                                                                                                depth_file_name='default', \n",
    "                                                                                                test_size_percentage=test_size_percentage)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5974955277280859, 0.46869409660107336, 0.007155635062611807, 0.6046511627906976]\n"
     ]
    }
   ],
   "source": [
    "print(class_acc_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_classification_rdf = class_acc_test[3]\n",
    "\n",
    "save_pickle(final_classification_rdf, \"metrics/\" + dataset_string + \"_rdf_classification_baseline.pkl\")\n",
    "save_pickle(class_report_rdf, \"label-metrics/\" + dataset_string + \"_class_report_baseline_trials.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gen-methods",
   "language": "python",
   "name": "gen-methods"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}