Baseline - LSTM-VAE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ee697e47-e8e0-4026-b895-3e9b7ee1264e",
   "metadata": {},
   "source": [
    "Ref: https://github.com/SchindlerLiang/VAE-for-Anomaly-Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a80a379b-932c-4401-8d76-b5ae5dedc6cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\";\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "os.environ[\"TF_FORCE_GPU_ALLOW_GROWTH\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "saving-behavior",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from evaluator import evaluate\n",
    "from data_loader import load_kdd_cup_urc, load_yahoo_A1, load_yahoo_A2, load_yahoo_A3, load_yahoo_A4, load_power_demand # Univariate Datasets\n",
    "from data_loader import load_nasa, load_ecg, load_gesture, load_smd # Multivariate Datasets\n",
    "\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "# THESE LINES ARE FOR REPRODUCIBILITY\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "tf.random.set_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "49c0a8e3-705d-429b-b7a0-c3c26a3a7b2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sampling(layers.Layer):\n",
    "    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n",
    "\n",
    "    def call(self, inputs):\n",
    "        z_mean, z_log_var = inputs\n",
    "        batch = tf.shape(z_mean)[0]\n",
    "        dim = tf.shape(z_mean)[1]\n",
    "        epsilon = tf.keras.backend.random_normal(shape=(batch, dim))\n",
    "        return z_mean + tf.exp(0.5 * z_log_var) * epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "24e86739-e767-4960-b7ea-9904d224a0d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(keras.Model):\n",
    "    def __init__(self, encoder, decoder, **kwargs):\n",
    "        super(VAE, self).__init__(**kwargs)\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n",
    "        self.reconstruction_loss_tracker = keras.metrics.Mean(\n",
    "            name=\"reconstruction_loss\"\n",
    "        )\n",
    "        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n",
    "\n",
    "    @property\n",
    "    def metrics(self):\n",
    "        return [\n",
    "            self.total_loss_tracker,\n",
    "            self.reconstruction_loss_tracker,\n",
    "            self.kl_loss_tracker,\n",
    "        ]\n",
    "\n",
    "    def train_step(self, data):\n",
    "        with tf.GradientTape() as tape:\n",
    "            z_mean, z_log_var, z = self.encoder(data)\n",
    "            reconstruction = self.decoder(z)\n",
    "            reconstruction_loss = tf.reduce_mean(\n",
    "                tf.reduce_sum(\n",
    "                    keras.losses.binary_crossentropy(data, reconstruction), axis=1\n",
    "                )\n",
    "            )\n",
    "            kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))\n",
    "            kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))\n",
    "            total_loss = reconstruction_loss + kl_loss\n",
    "        grads = tape.gradient(total_loss, self.trainable_weights)\n",
    "        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
    "        self.total_loss_tracker.update_state(total_loss)\n",
    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
    "        self.kl_loss_tracker.update_state(kl_loss)\n",
    "        return {\n",
    "            \"loss\": self.total_loss_tracker.result(),\n",
    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ac4f72c2-a0d7-481e-9993-fadd117b6a58",
   "metadata": {},
   "outputs": [],
   "source": [
    "def LSTM_VAE(X_train):\n",
    "    latent_dim = 16\n",
    "\n",
    "    encoder_inputs = keras.Input(shape=(X_train.shape[1], X_train.shape[2]))\n",
    "    x = layers.LSTM(64, return_sequences=True)(encoder_inputs)\n",
    "    x = layers.LSTM(32)(x)\n",
    "    z_mean = layers.Dense(latent_dim, name=\"z_mean\")(x)\n",
    "    z_log_var = layers.Dense(latent_dim, name=\"z_log_var\")(x)\n",
    "    z = Sampling()([z_mean, z_log_var])\n",
    "    encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name=\"encoder\")\n",
    "\n",
    "    latent_inputs = keras.Input(shape=(latent_dim,))\n",
    "    x = layers.RepeatVector(X_train.shape[1])(latent_inputs)\n",
    "    x = layers.LSTM(32, return_sequences=True)(x)\n",
    "    x = layers.LSTM(64)(x)\n",
    "    x = layers.Dense(X_train.shape[1] *  X_train.shape[2])(x)\n",
    "    decoder_outputs = layers.Reshape([X_train.shape[1], X_train.shape[2]])(x)\n",
    "    decoder = keras.Model(latent_inputs, decoder_outputs, name=\"decoder\")\n",
    "\n",
    "    model = VAE(encoder, decoder)\n",
    "    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001))\n",
    "    history = model.fit(X_train, epochs=50, batch_size=128, verbose=0)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa97c067-d18d-4c06-ae1b-2cbfe88788b6",
   "metadata": {},
   "source": [
    "### Yahoo S5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eight-partition",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71866d92-1163-4975-bb99-4302ccda66c3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5f894e9fa1b42e69958809a9960e5fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36707751a44042eb8fd5393c85baef5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A1 0.999999948484851 0.9848484768795222 0.9999999981470107\n",
      "yahoo_A1 0.9999999416666693 0.9166666376815787 0.9999999913849764\n",
      "yahoo_A1 0.9777777234567926 0.9514492572839868 0.999557629337239\n",
      "yahoo_A1 0.999999930000003 0.5999999693333345 0.9999999797237573\n",
      "yahoo_A1 0.9230768591716004 0.7922077522590122 0.9980158561333934\n",
      "yahoo_A1 0.9230768591716004 0.7647058436672041 0.9956349038267986\n",
      "yahoo_A1 0.999999948484851 0.9848484767788666 0.9999999981601732\n",
      "yahoo_A1 0.983870916961501 0.9838277861871575 0.991221484289207\n",
      "yahoo_A1 0.9322708656116595 0.9701074089636132 0.9877399217730798\n",
      "yahoo_A1 0.9999999487179513 0.9871794802015625 0.9999999983719279\n",
      "yahoo_A1 0.649999955475003 0.6814825280271881 0.709558802239739\n",
      "yahoo_A1 0.9937887686431877 0.9866027870087215 0.9996704874044167\n",
      "yahoo_A1 0.9999999483870992 0.9838709593548329 0.9999999980537635\n",
      "yahoo_A1 0.9230768686390559 0.7221973300069839 0.9930555497499013\n",
      "yahoo_A1 0.9999999487013013 0.9740259679924831 0.9999999983645984\n",
      "yahoo_A1 0.99999994761905 0.976190464740001 0.9999999973104057\n",
      "yahoo_A1 0.559055077611758 0.26965543794956975 0.3049788287200894\n",
      "yahoo_A1 0.40223459717237664 0.3102154414002211 0.7263409488658754\n",
      "yahoo_A1 0.49795914607247255 0.01728316223586466 0.00022583558908816095\n",
      "yahoo_A1 0.5101214192824038 0.00932283537910521 0.01689769451501404\n",
      "yahoo_A1 0.999999942307695 0.9230768956480069 0.9999999920306839\n",
      "yahoo_A1 0.9411764096885841 0.8428571090946728 0.9981735046174463\n",
      "yahoo_A1 0.816326480299878 0.8694435774900711 0.9364672166095267\n",
      "yahoo_A1 0.9246574838032492 0.9770742261030765 0.9839145859455207\n",
      "yahoo_A1 0.999999930000003 0.7999999503333362 0.9999999797237571\n",
      "yahoo_A1 0.580896644715756 0.17389013391202454 0.26353985328483226\n",
      "yahoo_A1 0.48780484043307837 0.33406184660594007 0.49209504842456864\n",
      "yahoo_A1 0.6338797316611463 0.4559258992317793 0.8262680318658836\n",
      "yahoo_A1 0.37861912281189325 0.08401121822894918 0.25851016495804474\n",
      "yahoo_A1 0.9999999472222247 0.97222220927827 0.9999999969154738\n",
      "yahoo_A1 0.7260578597665716 0.5614761159887169 0.6263648463094326\n",
      "yahoo_A1 0.9999999487179513 0.9871794802290861 0.9999999983707265\n",
      "yahoo_A1 0.9999999482758646 0.9827586106113487 0.9999999979532812\n",
      "yahoo_A1 0.38241755131989136 0.09042752737746064 0.34003234806649085\n",
      "yahoo_A1 0.9999999484375026 0.9843749918217852 0.9999999981085527\n",
      "yahoo_A1 0.49795914607247255 0.021506949712596143 0.03196834730359667\n",
      "yahoo_A1 0.37861912281189325 0.12503555519159681 0.42056299740557496\n",
      "yahoo_A1 0.1860464902109268 0.060621574115758005 0.9133656348505221\n",
      "yahoo_A1 0.37861912281189325 0.009988398172398444 0.09248455977957412\n",
      "yahoo_A1 0.534136506821667 0.013887589612606416 0.04100000839092907\n",
      "yahoo_A1 0.34146338616624533 0.01742328729801959 0.04908435068825932\n",
      "yahoo_A1 0.580896644715756 0.16904327908559785 0.26645226050983317\n",
      "yahoo_A1 0.580896644715756 0.042523112634846275 0.05927651638328743\n",
      "yahoo_A1 0.49795914607247255 0.002024293121831413 -0.0\n",
      "yahoo_A1 0.99999994761905 0.9761904644923128 0.9999999973104055\n",
      "yahoo_A1 0.9230768591716004 0.7182270847631614 0.9160714142480869\n",
      "yahoo_A1 0.9436619205911553 0.9454348540823041 0.9762328750492012\n",
      "yahoo_A1 0.0 0.0 -0.0\n",
      "yahoo_A1 0.4999999530000034 0.4339839716591485 0.8778027455836929\n",
      "yahoo_A1 0.050541511282566315 0.01218625751200981 0.5175694356836613\n",
      "yahoo_A1 0.9090908582809945 0.8920635808732895 0.9519181947104131\n",
      "yahoo_A1 0.9999999490384642 0.9909090850408736 0.9999999986508749\n",
      "yahoo_A1 0.9999999473684236 0.9736841974468501 0.999999997059779\n",
      "yahoo_A1 0.8023255306314255 0.8291435955592683 0.9418357287109945\n",
      "yahoo_A1 0.999999948571431 0.9857142781135123 0.9999999982289628\n",
      "yahoo_A1 0.2961275372792815 0.022702128009181966 0.053702208461516056\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "165a13bb5a7f4e9196abafa553f86eea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16124903f0c545d3a187ba57bf052462",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A2 0.2631578458141056 0.18128559747950035 0.7149426137982969\n",
      "yahoo_A2 0.17493470983952586 0.0045711629141228696 0.06687473854231647\n",
      "yahoo_A2 0.9999999485074653 0.9850746190039017 0.9999999983492349\n",
      "yahoo_A2 0.2791327670397565 0.14492929425996695 0.4158594605903446\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.9925925411248311 0.9842862917821513 0.999917342010811\n",
      "yahoo_A2 0.5565216966922947 0.3874606641070091 0.839705432514738\n",
      "yahoo_A2 0.0044247783176051365 0.0011086473390003108 0.02807775096585821\n",
      "yahoo_A2 0.5555555122470883 0.39996739275337734 0.9140964466300932\n",
      "yahoo_A2 0.5314684910558003 0.4287646423371254 0.5589108258609173\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.35294114533737286 0.2159137126993225 0.8055152670421571\n",
      "yahoo_A2 0.2482100020665201 0.08299421837866136 0.3368152696870141\n",
      "yahoo_A2 0.8634360729375721 0.9122917192098301 0.9874160550598852\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.17493470983952586 0.003051915299083975 0.051484989932421725\n",
      "yahoo_A2 0.9999999490384642 0.9903846098908912 0.9999999988797316\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.7085713805061257 0.6146644187091741 0.9600887949388296\n",
      "yahoo_A2 0.2482100020665201 0.02007355464755713 0.10310820379068372\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.9999999485074653 0.9850746190040898 0.9999999983492349\n",
      "yahoo_A2 0.2482100020665201 0.038530432326106416 0.08845534344661402\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.4999999375000028 0.16666664444444684 0.9956802457892797\n",
      "yahoo_A2 0.17983649584598738 0.03919731160729689 0.3295886405723713\n",
      "yahoo_A2 0.6862744591503305 0.6549964334985301 0.944055249247655\n",
      "yahoo_A2 0.0043668117894395606 0.0010940917940713256 0.016198702480302796\n",
      "yahoo_A2 0.6702702233455109 0.5771923580373483 0.9563456436130607\n",
      "yahoo_A2 0.9999999490384642 0.9903846098748423 0.9999999988797316\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.9999999485074653 0.9850746190043232 0.9999999983492349\n",
      "yahoo_A2 0.4840763877804414 0.35255669639530895 0.5142886137282303\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.17493470983952586 0.029494126492891203 0.14337409806623153\n",
      "yahoo_A2 0.20949261594284982 0.06631420858495979 0.46806176679564493\n",
      "yahoo_A2 0.9999999490384642 0.9903846098917983 0.9999999988797315\n",
      "yahoo_A2 0.9999998500000123 0.0 0.999999899784027\n",
      "yahoo_A2 0.8571428058567501 0.8525220433654944 0.9893963709342811\n",
      "yahoo_A2 0.9904761395374175 0.961271320406697 0.9975274714099739\n",
      "yahoo_A2 0.018691586898419075 0.0047169806559274234 0.7732180650597876\n",
      "yahoo_A2 0.7199999496088924 0.6530601562957398 0.9637374818778282\n",
      "yahoo_A2 0.7773278858397972 0.7179143648935651 0.9631172373521208\n",
      "yahoo_A2 0.004454342537983443 0.0011160713167151738 0.03455723195797929\n",
      "yahoo_A2 0.9999999485074653 0.9850746189963444 0.999999998349235\n",
      "yahoo_A2 0.3012820092435134 0.22380937585975152 0.6800936248395023\n",
      "yahoo_A2 0.3104574880270434 0.13057885483432283 0.5196199627878494\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.17493470983952586 0.01052570087200024 0.10324131814273116\n",
      "yahoo_A2 0.9999999490384642 0.990384609891736 0.9999999988797315\n",
      "yahoo_A2 0.006116207337579141 0.0015337421774436525 0.2980561256375718\n",
      "yahoo_A2 0.9999999485074653 0.9850746190025202 0.999999998349235\n",
      "yahoo_A2 0.3450585975786269 0.2037374316005605 0.6188110493678155\n",
      "yahoo_A2 0.04347825642722119 0.011111109975308755 0.9049675118995839\n",
      "yahoo_A2 0.208809115861076 0.0852503365294371 0.369421769995259\n",
      "yahoo_A2 0.6075084862164999 0.654635721102008 0.9239850416999312\n",
      "yahoo_A2 0.004347825651228733 0.0010893245095666068 0.010799134986868566\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.28571423980196353 0.1680706010031076 0.7226371940567793\n",
      "yahoo_A2 0.9999999490384642 0.9903846098891442 0.9999999988797315\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.17493470983952586 0.004192032409054522 0.03252581472958236\n",
      "yahoo_A2 0.9999999490384642 0.990384609886736 0.9999999988797315\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.9999999485074653 0.9850746189974395 0.9999999983492349\n",
      "yahoo_A2 0.2482100020665201 0.044708215227075354 0.4721210042685129\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746190007603 0.9999999983492349\n",
      "yahoo_A2 0.9999999485074653 0.985074618997327 0.9999999983492349\n",
      "yahoo_A2 0.6821704939847398 0.5692380358775961 0.927342795053923\n",
      "yahoo_A2 0.007692306920118344 0.001930501736706389 0.444924361458984\n",
      "yahoo_A2 0.8211920025262079 0.7457692054387789 0.9833506501809889\n",
      "yahoo_A2 0.303951341074087 0.25642246851170003 0.5581043949791686\n",
      "yahoo_A2 0.39999995200000177 0.12499998437500164 0.993520418791906\n",
      "yahoo_A2 0.9999999485074653 0.9850746190042323 0.999999998349235\n",
      "yahoo_A2 0.9999999490384642 0.9903846098894069 0.9999999988797315\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.9999999485074653 0.9850746189994556 0.9999999983492349\n",
      "yahoo_A2 0.2482100020665201 0.005317880849968626 0.024401386006970234\n",
      "yahoo_A2 0.5277777369212994 0.42322252091516566 0.6090057091397054\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.17493470983952586 0.01573792929547205 0.08455082297851806\n",
      "yahoo_A2 0.9999999490384642 0.9903846098876362 0.9999999988797315\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746190034814 0.9999999983492349\n",
      "yahoo_A2 0.9999999490384642 0.9903846098842625 0.9999999988797316\n",
      "yahoo_A2 0.2222221975308647 0.06249999296875071 0.9848811108024111\n",
      "yahoo_A2 0.9999999485074653 0.9850746190026449 0.9999999983492349\n",
      "yahoo_A2 0.9999999490384642 0.9903846098917147 0.9999999988797316\n",
      "yahoo_A2 0.9999998500000123 0.0 0.999999899784027\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746189625067 0.9999999983492349\n",
      "yahoo_A2 0.2482100020665201 0.04556711792279238 0.05812972122957483\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.18010751044448928 0.032127221613167924 0.2379586763593546\n",
      "yahoo_A2 0.9999999490384642 0.9903846098916357 0.9999999988797315\n",
      "yahoo_A2 0.008163264486463973 0.002049180122111012 0.4762418529209028\n",
      "yahoo_A2 0.9999999485074653 0.985074619003901 0.999999998349235\n",
      "yahoo_A2 0.5818181314380209 0.6026701761166575 0.911347679076726\n",
      "yahoo_A2 0.004329003895166881 0.0010845985897864317 0.006479480992121074\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38bd46dad7fc4746b065efb44927546f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccaa3d74733b4062aa44b7159c5f90c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/92 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A3 0.25396823169564314 0.00872130222697063 0.0731309495821311\n",
      "yahoo_A3 0.442477841491114 0.15411338849427517 0.2756639238497847\n",
      "yahoo_A3 0.7249999504921909 0.6484591998728928 0.8486467699025526\n",
      "yahoo_A3 0.25396823169564314 0.00266806519236573 0.04220159812623962\n",
      "yahoo_A3 0.6314151976538852 0.0832432687580802 0.18515377867348662\n",
      "yahoo_A3 0.41726615387920135 0.006303056573007273 0.04302540240865644\n",
      "yahoo_A3 0.5852089616298454 0.19395608566245498 0.1849967069643177\n",
      "yahoo_A3 0.45614031550631234 0.07701259018435355 0.17775349453678133\n",
      "yahoo_A3 0.5210083646578659 0.017765224650788395 0.14423502063428945\n",
      "yahoo_A3 0.7134502462945892 0.4312674328519862 0.41706829109124843\n",
      "yahoo_A3 0.637770854195864 0.21691656961536682 0.25639030680153013\n",
      "yahoo_A3 0.5783521396394755 0.4634619675981355 0.46970549043255505\n",
      "yahoo_A3 0.19301846297450495 0.00142045541775027 0.10941475456995176\n",
      "yahoo_A3 0.6766916843462069 0.02697169209311013 0.032604072907543066\n",
      "yahoo_A3 0.11563168070466742 0.002571192338179495 0.07703916067105396\n",
      "yahoo_A3 0.6715685778726967 0.7227641869488662 0.777542602399393\n",
      "yahoo_A3 0.25396823169564314 0.002831390501729258 0.04589582166944692\n",
      "yahoo_A3 0.25396823169564314 0.07156487238556176 0.12333158916238915\n",
      "yahoo_A3 0.2812499757202169 0.00282871208192994 0.06455577978107209\n",
      "yahoo_A3 0.24849697212621819 0.0814681467376934 0.22331748532628656\n",
      "yahoo_A3 0.2258064314906365 0.007416907146327606 0.021816425515368137\n",
      "yahoo_A3 0.45070419027970904 0.001597445401434294 -0.0\n",
      "yahoo_A3 0.6075948942156735 0.0020080338483952297 -0.0\n",
      "yahoo_A3 0.4086798950024387 0.05380597699442023 0.6599253246869482\n",
      "yahoo_A3 0.25396823169564314 0.0013262609136845765 -0.0\n",
      "yahoo_A3 0.5185184799351569 0.3660038586686556 0.6685424043775482\n",
      "yahoo_A3 0.43137251503268237 0.25496744596755605 0.5810371179267656\n",
      "yahoo_A3 0.25396823169564314 0.0013262609052963194 -0.0\n",
      "yahoo_A3 0.3763837331436143 0.13758941620377405 0.3600847160018473\n",
      "yahoo_A3 0.637770854195864 0.05018547961720812 0.06187909515683371\n",
      "yahoo_A3 0.8916876077927048 0.6755665335922818 0.37829823822852143\n",
      "yahoo_A3 0.7726638296349493 0.041228483494578735 0.07006327810200402\n",
      "yahoo_A3 0.5357736710795402 0.10070437693356181 0.10090339082889078\n",
      "yahoo_A3 0.7428570959510233 0.7765189134889079 0.6950917927325827\n",
      "yahoo_A3 0.6075948942156735 0.2445571991200961 0.3504261988417927\n",
      "yahoo_A3 0.7356321371911775 0.02517233659469046 0.16427610242289062\n",
      "yahoo_A3 0.6075948942156735 0.46787853877956515 0.5474955226768435\n",
      "yahoo_A3 0.6228481572302212 0.09891955140652399 0.1376767383627144\n",
      "yahoo_A3 0.45070419027970904 0.16866120909286483 0.3592622694830724\n",
      "yahoo_A3 0.7978141594762488 0.8501723271312993 0.7845971916512016\n",
      "yahoo_A3 0.42767291778015437 0.2134257954360625 0.8338028473425778\n",
      "yahoo_A3 0.6958762383661956 0.7622193980384402 0.7691363875347544\n",
      "yahoo_A3 0.25396823169564314 0.0013262608999583332 -0.0\n",
      "yahoo_A3 0.323809496547848 0.05483561611983656 0.3025942633906634\n",
      "yahoo_A3 0.7153284209963267 0.02237635695318048 0.03535048828595764\n",
      "yahoo_A3 0.7743732113825961 0.03717179678366313 0.03758448705672889\n",
      "yahoo_A3 0.8282289792174155 0.4073535647133137 0.24689172526558328\n",
      "yahoo_A3 0.6846038411666818 0.3670041811618394 0.4645171862087267\n",
      "yahoo_A3 0.25396823169564314 0.07096041432849898 0.40386332077938697\n",
      "yahoo_A3 0.40579706895610423 0.2373808723852231 0.6045480815921032\n",
      "yahoo_A3 0.5121950721825679 0.4188371344036883 0.7296658338536722\n",
      "yahoo_A3 0.5874798941145886 0.025700570569390797 0.2775189797431433\n",
      "yahoo_A3 0.5259631101380745 0.014661888019893113 0.10262581878701219\n",
      "yahoo_A3 0.700147664776179 0.4136186543463577 0.5503588042615439\n",
      "yahoo_A3 0.7586206397146287 0.8023622627147404 0.8855368154517627\n",
      "yahoo_A3 0.9543567956887821 0.9351981888864763 0.9877461369738382\n",
      "yahoo_A3 0.540630142792948 0.08939692218514633 0.10805654817540399\n",
      "yahoo_A3 0.3802816410335313 0.28169432414284495 0.7177213592029779\n",
      "yahoo_A3 0.7283236529018707 0.06933160249753485 0.1297278309938754\n",
      "yahoo_A3 0.6646433545110223 0.059823541390844004 0.015369790910554008\n",
      "yahoo_A3 0.5405404949995077 0.4587304204869044 0.714132548844746\n",
      "yahoo_A3 0.5135134751734871 0.07028650836103889 0.1616746535206558\n",
      "yahoo_A3 0.25396823169564314 0.08976100293524288 0.2838062548975344\n",
      "yahoo_A3 0.6943619722688439 0.47625344495565697 0.5228168972554935\n",
      "yahoo_A3 0.700147664776179 0.32918228945484157 0.3211646051797875\n",
      "yahoo_A3 0.6726998043237837 0.7583435223920172 0.7221007587841312\n",
      "yahoo_A3 0.442477841491114 0.20310362259649908 0.48427893140908207\n",
      "yahoo_A3 0.8541666175075981 0.1219178683101242 0.057424375263739114\n",
      "yahoo_A3 0.7301586835887245 0.7000525889219968 0.6437701845807111\n",
      "yahoo_A3 0.25049699588078084 0.019116087285598746 0.04212198344486636\n",
      "yahoo_A3 0.45070419027970904 0.30618248285755767 0.513699683983597\n",
      "yahoo_A3 0.43694490352936993 0.07576052699572214 0.3227183960811624\n",
      "yahoo_A3 0.7578124501892122 0.7849501865391285 0.8976958632780935\n",
      "yahoo_A3 0.5357736710795402 0.249077963307643 0.5982289842824213\n",
      "yahoo_A3 0.4140624646301298 0.24509013837398735 0.7729659414888593\n",
      "yahoo_A3 0.25396823169564314 0.0013812164241038393 0.04122340131882383\n",
      "yahoo_A3 0.45070419027970904 0.01648759132677844 0.06843293588024575\n",
      "yahoo_A3 0.4144143814106024 0.004057995729821608 0.036547205282460436\n",
      "yahoo_A3 0.277886473028215 0.020810903189231192 0.22760508448669298\n",
      "yahoo_A3 0.6865671188861692 0.34585115443240005 0.38896160435190563\n",
      "yahoo_A3 0.6865671188861692 0.6041613949201817 0.6031761521695981\n",
      "yahoo_A3 0.5783521396394755 0.009954666963897503 0.026235359183999468\n",
      "yahoo_A3 0.6398763086823295 0.4634608090940947 0.5542943191990625\n",
      "yahoo_A3 0.7692307198271111 0.858521690872853 0.9229749318475506\n",
      "yahoo_A3 0.4454148121040815 0.13017170087829757 0.4632983603068117\n",
      "yahoo_A3 0.4879724715319875 0.010026017357751868 0.047409329960524105\n",
      "yahoo_A3 0.3301707502968744 0.0014367827016498764 0.016997165434278583\n",
      "yahoo_A3 0.7811633870811333 0.27636897934061755 0.09403045663966236\n",
      "yahoo_A3 0.3882783568463308 0.004815779728089145 0.0029940118254508873\n",
      "yahoo_A3 0.45070419027970904 0.11033810565233139 0.2770961432808739\n",
      "yahoo_A3 0.516020197625476 0.11716246042960166 0.22331023037282663\n",
      "yahoo_A3 0.5957446302229558 0.5332119871030128 0.7427480833953048\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "254b4a6501fe424ba0e039cc247819af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "975ec2ebe80b454eb4b88b966e48e137",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/82 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A4 0.4853700147102329 0.3399696426036133 0.573594045239534\n",
      "yahoo_A4 0.2544731387231303 0.01901853524344787 0.12147789490706809\n",
      "yahoo_A4 0.41726615387920135 0.006344003765696372 0.10679045566215023\n",
      "yahoo_A4 0.2945736181779962 0.03952241846650187 0.1972813760890338\n",
      "yahoo_A4 0.7228915165553815 0.81232225385025 0.8052707348018777\n",
      "yahoo_A4 0.7741934993743688 0.8500618710372089 0.8883061459251756\n",
      "yahoo_A4 0.47887318904098036 0.48156165407522905 0.6356954289518165\n",
      "yahoo_A4 0.3673469086517006 0.170447019733724 0.46949684383476864\n",
      "yahoo_A4 0.5942491593197877 0.24501287095184457 0.318006279858834\n",
      "yahoo_A4 0.45070419027970904 0.0069178572014325 0.04288103965973015\n",
      "yahoo_A4 0.637770854195864 0.3286424849875839 0.3758882345990707\n",
      "yahoo_A4 0.3882783568463308 0.005563924512424259 0.06608312302608893\n",
      "yahoo_A4 0.45070419027970904 0.0015974454027983312 -0.0\n",
      "yahoo_A4 0.3551401575753364 0.13028095441711915 0.3443509510055781\n",
      "yahoo_A4 0.07439824225732532 0.0011792459581944472 -0.0\n",
      "yahoo_A4 0.25396823169564314 0.07094858163552302 0.2956025448684697\n",
      "yahoo_A4 0.5942491593197877 0.1845609990499536 0.2619897479201431\n",
      "yahoo_A4 0.6075948942156735 0.020711600071494585 0.1176147813012523\n",
      "yahoo_A4 0.7675069552887851 0.4765147536250031 0.38689602650095034\n",
      "yahoo_A4 0.6075948942156735 0.046083067896928384 0.05955606547149972\n",
      "yahoo_A4 0.25396823169564314 0.005802139410514879 0.12336984315129662\n",
      "yahoo_A4 0.45070419027970904 0.07341634586611827 0.05943228745002024\n",
      "yahoo_A4 0.5714285304267189 0.43642882436887126 0.4567703363091714\n",
      "yahoo_A4 0.25396823169564314 0.006661215346531034 0.06237968248416301\n",
      "yahoo_A4 0.8906249493316679 0.8869779779869149 0.9734345638649665\n",
      "yahoo_A4 0.5454545056048113 0.04023228753631156 0.13647206598364614\n",
      "yahoo_A4 0.8328911978596936 0.12977775906284972 0.04879369775023624\n",
      "yahoo_A4 0.25396823169564314 0.031932681660054724 0.08997138263872886\n",
      "yahoo_A4 0.5897435479700884 0.4921515611897072 0.593253243295224\n",
      "yahoo_A4 0.7726638296349493 0.49067889813420484 0.4797157463453538\n",
      "yahoo_A4 0.8837208761492726 0.8610717358924577 0.9934083368894231\n",
      "yahoo_A4 0.4086798950024387 0.08919827604317666 0.31000862551656444\n",
      "yahoo_A4 0.25396823169564314 0.0013262609024772653 -0.0\n",
      "yahoo_A4 0.42293903459873594 0.011268912031838907 0.06311994679738257\n",
      "yahoo_A4 0.6666666220202049 0.32381772118587837 0.3899329347896036\n",
      "yahoo_A4 0.5927151898858631 0.3662514870882821 0.5012802091041176\n",
      "yahoo_A4 0.5620914626724793 0.09903828369905751 0.2538067068540561\n",
      "yahoo_A4 0.6075948942156735 0.36519313518066043 0.5133611914445478\n",
      "yahoo_A4 0.4748699809545005 0.13611568401458457 0.28454074984103384\n",
      "yahoo_A4 0.3826086572703251 0.2804875892678105 0.3547187725260642\n",
      "yahoo_A4 0.557377008793338 0.48781706853877305 0.6426400724682675\n",
      "yahoo_A4 0.03131990741758405 0.0012195125386435899 0.05542724669620665\n",
      "yahoo_A4 0.5806451198959446 0.2940673574609912 0.5844714285557118\n",
      "yahoo_A4 0.3011582754610119 0.05723569280033196 0.19049872821417133\n",
      "yahoo_A4 0.3703703400548721 0.049966128126160116 0.23368384752560076\n",
      "yahoo_A4 0.5147058324718905 0.3630116279374379 0.8347018298648106\n",
      "yahoo_A4 0.6091370091164456 0.5686771209563677 0.5629299878940249\n",
      "yahoo_A4 0.25396823169564314 0.05348598557584157 0.42839211789931925\n",
      "yahoo_A4 0.46422334988015346 0.00808107954268001 0.014561468755605348\n",
      "yahoo_A4 0.45070419027970904 0.004348337053310133 0.02034412278935244\n",
      "yahoo_A4 0.7227866009475067 0.020374584870582763 0.03561991507461088\n",
      "yahoo_A4 0.6898733677976321 0.7688176377777852 0.7662317666886777\n",
      "yahoo_A4 0.25396823169564314 0.0013262609029890953 -0.0\n",
      "yahoo_A4 0.9142856633469416 0.8971217086491649 0.9903174848913744\n",
      "yahoo_A4 0.5852089616298454 0.011438796493941972 0.04456191793756426\n",
      "yahoo_A4 0.5270269820215523 0.5563171940662468 0.7279994734276468\n",
      "yahoo_A4 0.4974618800072195 0.48316231787438774 0.5859828628370214\n",
      "yahoo_A4 0.2978723149594655 0.03102991945611772 0.09453571012679618\n",
      "yahoo_A4 0.43781090179946464 0.25424446145919427 0.7274606306035476\n",
      "yahoo_A4 0.5919999581337628 0.027201693178503215 0.04864784734192236\n",
      "yahoo_A4 0.45070419027970904 0.039681007706118876 0.05227903346429903\n",
      "yahoo_A4 0.6982248064178453 0.018218024221351815 0.059733763031883345\n",
      "yahoo_A4 0.6066350205251495 0.6636094747189281 0.7388470392398261\n",
      "yahoo_A4 0.674698750268547 0.021003831870995283 0.05416729709783363\n",
      "yahoo_A4 0.25396823169564314 0.006870386738728107 0.03577716819916212\n",
      "yahoo_A4 0.4957264582716078 0.2619936215290245 0.4720445266492055\n",
      "yahoo_A4 0.7739463100189401 0.7355797723215789 0.9203708266506891\n",
      "yahoo_A4 0.25396823169564314 0.0420487866862349 0.036394704296919135\n",
      "yahoo_A4 0.5919999581337628 0.22899797551858744 0.33206260184988207\n",
      "yahoo_A4 0.6544342065351805 0.16367459590242003 0.15483977688340456\n",
      "yahoo_A4 0.25049699588078084 0.0013966489944739302 0.05305039255887316\n",
      "yahoo_A4 0.7535410293108873 0.602391593201214 0.4874196907465922\n",
      "yahoo_A4 0.5897435479700884 0.030368107830426976 0.08033891372352422\n",
      "yahoo_A4 0.6544342065351805 0.018079108387930146 0.03308075927554848\n",
      "yahoo_A4 0.45070419027970904 0.2107361132282599 0.4926607281490806\n",
      "yahoo_A4 0.6075948942156735 0.23906749749181574 0.37460615662023083\n",
      "yahoo_A4 0.45070419027970904 0.006133168946011597 0.02703299198867666\n",
      "yahoo_A4 0.35205989595309467 0.0029639745905879894 0.03279492012983595\n",
      "yahoo_A4 0.45070419027970904 0.11399498477866767 0.14245235344684792\n",
      "yahoo_A4 0.6626139372529846 0.5682928135310105 0.7139358026611686\n",
      "yahoo_A4 0.6419752648529217 0.09063234076917337 0.05757935090708371\n",
      "yahoo_A4 0.2608695424299727 0.007447784376274092 0.1352559500521362\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_yahoo_A1, load_yahoo_A2, load_yahoo_A3, load_yahoo_A4]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "    \n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bd9e37fd-124c-4f5c-8f1f-a9d61a4b0315",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>yahoo_A1</th>\n",
       "      <td>0.741957</td>\n",
       "      <td>0.591924</td>\n",
       "      <td>0.706386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A2</th>\n",
       "      <td>0.576314</td>\n",
       "      <td>0.386187</td>\n",
       "      <td>0.679169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A3</th>\n",
       "      <td>0.527801</td>\n",
       "      <td>0.230681</td>\n",
       "      <td>0.331648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A4</th>\n",
       "      <td>0.505855</td>\n",
       "      <td>0.224112</td>\n",
       "      <td>0.315003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                f1    pr_auc   roc_auc\n",
       "dataset                               \n",
       "yahoo_A1  0.741957  0.591924  0.706386\n",
       "yahoo_A2  0.576314  0.386187  0.679169\n",
       "yahoo_A3  0.527801  0.230681  0.331648\n",
       "yahoo_A4  0.505855  0.224112  0.315003"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yahoo_results = pd.DataFrame(total_scores)\n",
    "yahoo_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb09ed4f-3e8f-4260-8996-a6811c7734be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8890a678-6116-4222-b0ff-395fcd7f41aa",
   "metadata": {},
   "source": [
    "### NASA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e4387a80-585a-488e-94a8-546f42d47632",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "106a238d-c5cb-49e9-9764-b1bc37f09436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e49bb969be94423984ea8f3aa8e6c32f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dc253f8d01f4d608c9b2e5ddd83675b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D1 0.25300017462469454 0.1310774099523998 0.35531824696954745\n",
      "D2 0.2699530282282525 0.5840009300457375 0.5173913038147389\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_nasa]:\n",
    "    datasets = loader(100, 100)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "        \n",
    "        total_scores['dataset'].append(f'D{i+1}')\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(f'D{i+1}', np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "abb7a6a5-3924-4a33-bc54-871559819602",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1</th>\n",
       "      <td>0.253000</td>\n",
       "      <td>0.131077</td>\n",
       "      <td>0.355318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2</th>\n",
       "      <td>0.269953</td>\n",
       "      <td>0.584001</td>\n",
       "      <td>0.517391</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "D1       0.253000  0.131077  0.355318\n",
       "D2       0.269953  0.584001  0.517391"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nasa_results = pd.DataFrame(total_scores)\n",
    "nasa_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff85b8cf-8aa0-4712-8ddb-01d63ec0f497",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "41754734-5c75-4caf-9069-fb3b39c7a0da",
   "metadata": {},
   "source": [
    "### SMD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "60a26807-8359-42d4-82b4-9e1d37b6acaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "41af318e-1b08-4cc9-a680-6b4a978544f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ee17a678f9248f8babc76433b9ff75f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a98cd3e4b4f64c74bd5485f8b85cd7fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smd 0.20232181144029696 0.5556857798256385 0.521888680406877\n",
      "smd 0.7080223384287494 0.551710535514189 0.9191231912768012\n",
      "smd 0.24475759898047408 0.219950937840305 0.5903531431563934\n",
      "smd 0.26386073698048373 0.2890652644733706 0.5956179787098114\n",
      "smd 0.689655126355723 0.7416842166394192 0.9499106269538564\n",
      "smd 0.726425765802517 0.8022758852529401 0.8847852533178482\n",
      "smd 0.38105170869074706 0.3943878784882371 0.6394658963354369\n",
      "smd 0.2918424327107888 0.29273626205822867 0.7250530178185837\n",
      "smd 0.5189747196290893 0.5763596055986044 0.8654698589516957\n",
      "smd 0.2596273980668859 0.30899521959368986 0.5436930778435464\n",
      "smd 0.7339955349140422 0.7779197082812668 0.8677092922942592\n",
      "smd 0.24600048948742329 0.26278667750039353 0.5396083039471353\n",
      "smd 0.5496182710351756 0.5249348268195897 0.8358061982424857\n",
      "smd 0.5189553099967452 0.5240781476950535 0.7110228143980261\n",
      "smd 0.5284749841687082 0.5886110064415466 0.9159807520769212\n",
      "smd 0.9677418850899386 0.6730995648261773 0.9809677285777375\n",
      "smd 0.7653086246214532 0.8082500294945196 0.8200106460677644\n",
      "smd 0.5670497675463134 0.5878472277597009 0.6981995978023406\n",
      "smd 0.6648141259303697 0.6159987845354133 0.8491378288943825\n",
      "smd 0.7834710264654082 0.5823359537400504 0.9502762768277175\n",
      "smd 0.21122817449519962 0.14914135066891798 0.6600204477132212\n",
      "smd 0.21729485181328714 0.12068263093466747 0.5421121986022874\n",
      "smd 0.7725687228235574 0.6904396628892839 0.8756755504783792\n",
      "smd 0.36183391000923354 0.22189367865105378 0.7050552224739559\n",
      "smd 0.8197139439171299 0.8434328973108025 0.984888888099747\n",
      "smd 0.2230130287683529 0.33435914474618195 0.6684190708355718\n",
      "smd 0.4900432482100071 0.5188733130564858 0.8637362795003725\n",
      "smd 0.5603112435676573 0.49508316553532744 0.6958364677916222\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_smd]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "23474773-bac1-4145-ac75-c1a686bca5c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>smd</th>\n",
       "      <td>0.509571</td>\n",
       "      <td>0.501879</td>\n",
       "      <td>0.764279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "smd      0.509571  0.501879  0.764279"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smd_results = pd.DataFrame(total_scores)\n",
    "smd_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7599a9b-3b12-47e4-9d93-c48c00bb233d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "191def0b-8e38-4401-be4c-0d47e89eefa5",
   "metadata": {},
   "source": [
    "### ECG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9739a0bc-1fd1-48bc-86ad-d2fd75473bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cd9f21c4-27f5-412f-a213-60fc3c09acc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "224d6af90bde488db71fd20a52b39150",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5d1ef08dfa146efb745ec7c075bfccf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D1 0.6206896038049976 0.5326525936008679 0.6750068037272775\n",
      "D2 0.363636315151521 0.23541899528696025 0.5590456747449235\n",
      "D3 0.11320751384835404 0.04367631207943112 0.44307798338600746\n",
      "D4 0.41666661458333903 0.2013686028274416 0.6715630806541828\n",
      "WARNING:tensorflow:5 out of the last 17 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f689477bdd0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "D5 0.3999999612839539 0.16615675062595867 0.67033798814393\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f695036ab90> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "D6 0.31249995117188206 0.15095157139101562 0.6854580546040853\n",
      "D7 0.04918032266275918 0.01636272071549821 0.405622129457916\n",
      "D8 0.1951219282702121 0.09269561477001245 0.4519237190097242\n",
      "D9 0.38805966963689265 0.17316629517967963 0.37851313122196206\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_ecg]:\n",
    "    datasets = loader(32, 16)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(f'D{i+1}')\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(f'D{i+1}', np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4d23a70d-851c-497b-81ed-63c1369f4619",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1</th>\n",
       "      <td>0.620690</td>\n",
       "      <td>0.532653</td>\n",
       "      <td>0.675007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2</th>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.235419</td>\n",
       "      <td>0.559046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D3</th>\n",
       "      <td>0.113208</td>\n",
       "      <td>0.043676</td>\n",
       "      <td>0.443078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D4</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.201369</td>\n",
       "      <td>0.671563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D5</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.166157</td>\n",
       "      <td>0.670338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D6</th>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.150952</td>\n",
       "      <td>0.685458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D7</th>\n",
       "      <td>0.049180</td>\n",
       "      <td>0.016363</td>\n",
       "      <td>0.405622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D8</th>\n",
       "      <td>0.195122</td>\n",
       "      <td>0.092696</td>\n",
       "      <td>0.451924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D9</th>\n",
       "      <td>0.388060</td>\n",
       "      <td>0.173166</td>\n",
       "      <td>0.378513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "D1       0.620690  0.532653  0.675007\n",
       "D2       0.363636  0.235419  0.559046\n",
       "D3       0.113208  0.043676  0.443078\n",
       "D4       0.416667  0.201369  0.671563\n",
       "D5       0.400000  0.166157  0.670338\n",
       "D6       0.312500  0.150952  0.685458\n",
       "D7       0.049180  0.016363  0.405622\n",
       "D8       0.195122  0.092696  0.451924\n",
       "D9       0.388060  0.173166  0.378513"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ecg_results = pd.DataFrame(total_scores)\n",
    "ecg_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc56ec31-cb2d-4e45-bc21-84166bad9ac8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b153349b-4e8c-45d3-90cd-a1c765412ee0",
   "metadata": {},
   "source": [
    "### Power Demand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d38db439-5db0-4021-bc17-5f65f36f3f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7a1b4d2f-a3e8-4148-9590-eb9fa2a7def8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a99dedb287b429dbbc1bba486135806",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "power_demand 0.3606206231571944 0.10335052401683913 0.1360581420800021\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_power_demand]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "95679aba-14a8-4b08-aae0-27ce1f3b6407",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>power_demand</th>\n",
       "      <td>0.360621</td>\n",
       "      <td>0.103351</td>\n",
       "      <td>0.136058</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    f1    pr_auc   roc_auc\n",
       "dataset                                   \n",
       "power_demand  0.360621  0.103351  0.136058"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power_results = pd.DataFrame(total_scores)\n",
    "power_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64350634-7bfa-4fdf-8e15-c8e2998569a5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "06b9409f-6b54-4ee1-9079-850794f4269a",
   "metadata": {},
   "source": [
    "### 2D Gesture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e787152b-f6d3-4ce7-a7b6-cbad809e2d9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8640d9ef-f1a3-40cd-ab83-c9e0c6cf561e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c99a87fcafeb44f3a423879e675df490",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gesture 0.4203232924120907 0.19432085097920657 0.33302161234566896\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_gesture]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "\n",
    "        model = LSTM_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "08d069d1-8830-4cc9-8e5b-2b8f48e0dc3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gesture</th>\n",
       "      <td>0.420323</td>\n",
       "      <td>0.194321</td>\n",
       "      <td>0.333022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "gesture  0.420323  0.194321  0.333022"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gesture_results = pd.DataFrame(total_scores)\n",
    "gesture_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64c54c1d-6ebf-485d-9752-515479c9a3b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}