Baseline - CNN-VAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a80a379b-932c-4401-8d76-b5ae5dedc6cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\";\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "os.environ[\"TF_FORCE_GPU_ALLOW_GROWTH\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "saving-behavior",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from evaluator import evaluate\n",
    "from data_loader import load_kdd_cup_urc, load_yahoo_A1, load_yahoo_A2, load_yahoo_A3, load_yahoo_A4, load_power_demand # Univariate Datasets\n",
    "from data_loader import load_nasa, load_ecg, load_gesture, load_smd # Multivariate Datasets\n",
    "\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "# THESE LINES ARE FOR REPRODUCIBILITY\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "tf.random.set_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "49c0a8e3-705d-429b-b7a0-c3c26a3a7b2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sampling(layers.Layer):\n",
    "    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n",
    "\n",
    "    def call(self, inputs):\n",
    "        z_mean, z_log_var = inputs\n",
    "        batch = tf.shape(z_mean)[0]\n",
    "        dim = tf.shape(z_mean)[1]\n",
    "        epsilon = tf.keras.backend.random_normal(shape=(batch, dim))\n",
    "        return z_mean + tf.exp(0.5 * z_log_var) * epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "24e86739-e767-4960-b7ea-9904d224a0d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(keras.Model):\n",
    "    def __init__(self, encoder, decoder, **kwargs):\n",
    "        super(VAE, self).__init__(**kwargs)\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n",
    "        self.reconstruction_loss_tracker = keras.metrics.Mean(\n",
    "            name=\"reconstruction_loss\"\n",
    "        )\n",
    "        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n",
    "\n",
    "    @property\n",
    "    def metrics(self):\n",
    "        return [\n",
    "            self.total_loss_tracker,\n",
    "            self.reconstruction_loss_tracker,\n",
    "            self.kl_loss_tracker,\n",
    "        ]\n",
    "\n",
    "    def train_step(self, data):\n",
    "        with tf.GradientTape() as tape:\n",
    "            z_mean, z_log_var, z = self.encoder(data)\n",
    "            reconstruction = self.decoder(z)\n",
    "            reconstruction_loss = tf.reduce_mean(\n",
    "                tf.reduce_sum(\n",
    "                    keras.losses.binary_crossentropy(data, reconstruction), axis=1\n",
    "                )\n",
    "            )\n",
    "            kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))\n",
    "            kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))\n",
    "            total_loss = reconstruction_loss + kl_loss\n",
    "        grads = tape.gradient(total_loss, self.trainable_weights)\n",
    "        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
    "        self.total_loss_tracker.update_state(total_loss)\n",
    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
    "        self.kl_loss_tracker.update_state(kl_loss)\n",
    "        return {\n",
    "            \"loss\": self.total_loss_tracker.result(),\n",
    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25e36252-2bfc-4f40-9e62-ada4d1878573",
   "metadata": {},
   "outputs": [],
   "source": [
    "def CNN_VAE(X_train):\n",
    "    latent_dim = 16\n",
    "\n",
    "    encoder_inputs = keras.Input(shape=(X_train.shape[1], X_train.shape[2]))\n",
    "    x = layers.Conv1D(32, 3, activation=\"relu\", strides=2, padding=\"same\")(encoder_inputs)\n",
    "    x = layers.Conv1D(64, 3, activation=\"relu\", strides=2, padding=\"same\")(x)\n",
    "    x = layers.Flatten()(x)\n",
    "    x = layers.Dense(128, activation=\"relu\")(x)\n",
    "    z_mean = layers.Dense(latent_dim, name=\"z_mean\")(x)\n",
    "    z_log_var = layers.Dense(latent_dim, name=\"z_log_var\")(x)\n",
    "    z = Sampling()([z_mean, z_log_var])\n",
    "    encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name=\"encoder\")\n",
    "\n",
    "    latent_inputs = keras.Input(shape=(latent_dim,))\n",
    "    x = layers.Dense(32* 64, activation=\"relu\")(latent_inputs)\n",
    "    x = layers.Reshape((32, 64))(x)\n",
    "    x = layers.Conv1DTranspose(64, 3, activation=\"relu\", strides=2, padding=\"same\")(x)\n",
    "    x = layers.Conv1DTranspose(32, 3, activation=\"relu\", strides=2, padding=\"same\")(x)\n",
    "    x = layers.Flatten()(x)\n",
    "    x = layers.Dense(X_train.shape[1] * X_train.shape[2])(x)\n",
    "    decoder_outputs = layers.Reshape([X_train.shape[1], X_train.shape[2]])(x)\n",
    "    decoder = keras.Model(latent_inputs, decoder_outputs, name=\"decoder\")\n",
    "\n",
    "    model = VAE(encoder, decoder)\n",
    "    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001))\n",
    "    history = model.fit(X_train, epochs=50, batch_size=128, verbose=0)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa97c067-d18d-4c06-ae1b-2cbfe88788b6",
   "metadata": {},
   "source": [
    "### Yahoo S5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eight-partition",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71866d92-1163-4975-bb99-4302ccda66c3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41c2d4e6d5604358b0fc1b204a3df423",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c552f8c0b5c042c8b82d447c5df2fc21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A1 0.9850745754065519 0.983943807889094 0.9997952479426399\n",
      "yahoo_A1 0.9999999416666693 0.9166666376815789 0.9999999913849764\n",
      "yahoo_A1 0.9999999456521764 0.9565217209833222 0.9999999953614762\n",
      "yahoo_A1 0.999999930000003 0.7999999500000029 0.9999999797237571\n",
      "yahoo_A1 0.9230768591716004 0.7255952010491444 0.9700396684125173\n",
      "yahoo_A1 0.9230768591716004 0.7330320327944443 0.98293649362152\n",
      "yahoo_A1 0.999999948484851 0.9242424185659951 0.9999999981601732\n",
      "yahoo_A1 0.9999999492063519 0.9959999948774263 0.9999999987844084\n",
      "yahoo_A1 0.9999999492063519 0.9920634873720409 0.9999999987844084\n",
      "yahoo_A1 0.9999999487179513 0.9871794799338866 0.999999998371928\n",
      "yahoo_A1 0.9781021390777371 0.9830758977051822 0.9987592617322397\n",
      "yahoo_A1 0.9938649794572648 0.9875028275777142 0.9999560636088743\n",
      "yahoo_A1 0.9999999483870992 0.9838709593720104 0.9999999980537634\n",
      "yahoo_A1 0.999999944444447 0.9444444226958288 0.9999999941537467\n",
      "yahoo_A1 0.9999999487013013 0.9870129799870717 0.9999999983645982\n",
      "yahoo_A1 0.99999994761905 0.976190464783117 0.9999999973104056\n",
      "yahoo_A1 0.559055077611758 0.22516177981065272 0.06910505664728762\n",
      "yahoo_A1 0.44628094100130333 0.42772772666909215 0.796909028558557\n",
      "yahoo_A1 0.49795914607247255 0.08458177845799594 0.2709539791031311\n",
      "yahoo_A1 0.5101214192824038 0.06047686967407431 0.0627964827653421\n",
      "yahoo_A1 0.999999942307695 0.9230768956557769 0.9999999920306841\n",
      "yahoo_A1 0.9999999388888916 0.888888853752206 0.9999999886149166\n",
      "yahoo_A1 0.8973383525683498 0.9645137907842851 0.9781281029666943\n",
      "yahoo_A1 0.8635096997540394 0.8564907553414038 0.9212324852316687\n",
      "yahoo_A1 0.999999930000003 0.5999999693333345 0.9999999797237571\n",
      "yahoo_A1 0.9931972282359227 0.9927695902687366 0.999638377155289\n",
      "yahoo_A1 0.6007325544794679 0.4553069554484776 0.733379603823592\n",
      "yahoo_A1 0.8977272217329575 0.9447183135477262 0.970795835213476\n",
      "yahoo_A1 0.9523809012471681 0.9757037403036377 0.9963106015643757\n",
      "yahoo_A1 0.9999999472222247 0.9722222080632719 0.9999999969154737\n",
      "yahoo_A1 0.7965367477670987 0.48987464240345224 0.5622358560614203\n",
      "yahoo_A1 0.9999999487179513 0.9743589682897522 0.9999999983707266\n",
      "yahoo_A1 0.9999999482758646 0.9827586118416458 0.9999999979532813\n",
      "yahoo_A1 0.48717943645628375 0.517823251394417 0.6843631688809484\n",
      "yahoo_A1 0.9999999484375026 0.9843749918323117 0.9999999981085527\n",
      "yahoo_A1 0.49795914607247255 0.3717869586116618 0.6552562339993655\n",
      "yahoo_A1 0.7738094730654794 0.8253450208838411 0.9431036044726654\n",
      "yahoo_A1 0.9999999333333361 0.6666666369444455 0.9999999830563251\n",
      "yahoo_A1 0.9454544943544563 0.9732829361088936 0.9944672654492004\n",
      "yahoo_A1 0.980842861144143 0.9825716560278642 0.9920599393369387\n",
      "yahoo_A1 0.9999999487013013 0.9870129799889835 0.9999999983645983\n",
      "yahoo_A1 0.6614172758974551 0.7476552163810848 0.8432307315459165\n",
      "yahoo_A1 0.6995515195077354 0.7871718237817185 0.8586045693941001\n",
      "yahoo_A1 0.49795914607247255 0.0020242931179229873 -0.0\n",
      "yahoo_A1 0.9999999475000025 0.9761904640502026 0.9999999972508817\n",
      "yahoo_A1 0.9230768591716004 0.7307752914707588 0.9817460169589634\n",
      "yahoo_A1 0.9863013185213011 0.9809425853176892 0.9981306289490954\n",
      "yahoo_A1 0.0 0.0 -0.0\n",
      "yahoo_A1 0.9999999428571453 0.9285714025257789 0.9999999921626985\n",
      "yahoo_A1 0.2962962556927346 0.09074880909467231 0.8910052780004829\n",
      "yahoo_A1 0.7589285207509598 0.8273334290955614 0.9190650513428223\n",
      "yahoo_A1 0.9999999490384642 0.9909090856242008 0.9999999986508747\n",
      "yahoo_A1 0.9999999473684236 0.9473684105571164 0.999999997059779\n",
      "yahoo_A1 0.9900989589255981 0.9894719635704661 0.999803617358786\n",
      "yahoo_A1 0.999999948571431 0.9857142778686142 0.9999999982289627\n",
      "yahoo_A1 0.9999999484615409 0.9846153764903719 0.999999998137914\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a01bb8494144153881b56c3628889e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9391cec3d8c449899d95df4a536fac7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A2 0.17493470983952586 0.060955292515711496 0.4674302173967504\n",
      "yahoo_A2 0.17493470983952586 0.060981070878588325 0.433231042285094\n",
      "yahoo_A2 0.9925925411248311 0.9848534902316813 0.9999763822525424\n",
      "yahoo_A2 0.4745762214880833 0.45689242655734275 0.8403752791177312\n",
      "yahoo_A2 0.005025125124365547 0.0012594457175669027 0.1447084088240384\n",
      "yahoo_A2 0.9999999485074653 0.9850746190040223 0.9999999983492351\n",
      "yahoo_A2 0.8888888380312286 0.9508090230785602 0.9930250294125758\n",
      "yahoo_A2 0.004926107880560072 0.0012345677774729584 0.1274297928450488\n",
      "yahoo_A2 0.45414842982399645 0.3309141839260608 0.8735712247056815\n",
      "yahoo_A2 0.5405404980277607 0.570237431465294 0.8153107753419673\n",
      "yahoo_A2 0.666666577777783 0.2499999625000044 0.9978400727866534\n",
      "yahoo_A2 0.8905108976290721 0.9433253098908769 0.9950756843640398\n",
      "yahoo_A2 0.3287670732220005 0.30705705134089833 0.5907430935609866\n",
      "yahoo_A2 0.8113207039693873 0.8718071010363649 0.9731990221087788\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840274\n",
      "yahoo_A2 0.17493470983952586 0.014506128906800647 0.08931709749166655\n",
      "yahoo_A2 0.9999999490384642 0.9903846098922058 0.9999999988797315\n",
      "yahoo_A2 0.004301074837784714 0.0010775860989057032 -0.0\n",
      "yahoo_A2 0.9999999485074653 0.9850746189966499 0.9999999983492348\n",
      "yahoo_A2 0.32276653227915303 0.23302411129359052 0.7024483662820121\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9925925411248311 0.9848534902324956 0.9999763822525424\n",
      "yahoo_A2 0.51028801646091 0.4735377169390632 0.8666119873601477\n",
      "yahoo_A2 0.01388888749035494 0.0034965031444080747 0.6933044661569605\n",
      "yahoo_A2 0.09999998950000007 0.02631578670360138 0.9611230138313003\n",
      "yahoo_A2 0.22784808016245625 0.10766064362351283 0.499465113843753\n",
      "yahoo_A2 0.8410255903852758 0.9052755541706697 0.9854853468813397\n",
      "yahoo_A2 0.005617976964714052 0.001408450562983549 0.2354211427137342\n",
      "yahoo_A2 0.19504642196561037 0.05727326823702358 0.48374846532658783\n",
      "yahoo_A2 0.9999999490384642 0.9903846098919019 0.9999999988797316\n",
      "yahoo_A2 0.004750593348040238 0.0011904760711451364 0.09503238788444318\n",
      "yahoo_A2 0.9999999485074653 0.9850746189991083 0.9999999983492349\n",
      "yahoo_A2 0.2482100020665201 0.07574657824679717 0.2571778851857195\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.2867132529536936 0.1618053701278382 0.741129950672766\n",
      "yahoo_A2 0.17493470983952586 0.036454454593257135 0.24308759052205264\n",
      "yahoo_A2 0.9999999490384642 0.9903846097298334 0.9999999988797316\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746190045145 0.9999999983492349\n",
      "yahoo_A2 0.9951690311652572 0.9902023236229637 0.9999694738492406\n",
      "yahoo_A2 0.04166666241319446 0.01063829678587607 0.9017277714035233\n",
      "yahoo_A2 0.9197079778784193 0.9465290886942619 0.9908251447841803\n",
      "yahoo_A2 0.9999999490384642 0.9903846098920708 0.9999999988797315\n",
      "yahoo_A2 0.007434943491659872 0.0018656714545277528 0.4287256589786812\n",
      "yahoo_A2 0.9999999485074653 0.985074619003295 0.999999998349235\n",
      "yahoo_A2 0.9292928784001658 0.9600614805014676 0.993652069064124\n",
      "yahoo_A2 0.2482100020665201 0.061875460008626454 0.08987771887364802\n",
      "yahoo_A2 0.9999998500000123 0.0 0.999999899784027\n",
      "yahoo_A2 0.17493470983952586 0.023080581979188584 0.22630748542131462\n",
      "yahoo_A2 0.9999999490384642 0.9903846098832005 0.9999999988797316\n",
      "yahoo_A2 0.005305039255887257 0.0013297871007101761 0.19006477576888636\n",
      "yahoo_A2 0.9999999485074653 0.9850746190043917 0.9999999983492349\n",
      "yahoo_A2 0.975845359686343 0.986526792883787 0.9992826606631969\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.969696918239213 0.9735140818536214 0.9978981657435982\n",
      "yahoo_A2 0.9999999490384642 0.9903846098916045 0.9999999988797315\n",
      "yahoo_A2 0.33333329444444565 0.09999998800000123 0.9913605917945324\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.8571428058567501 0.9055660544838745 0.9852045137713539\n",
      "yahoo_A2 0.9811320245639044 0.9881809095586642 0.9996489610290867\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.17493470983952586 0.024367080493114424 0.14074785855301705\n",
      "yahoo_A2 0.9999999490384642 0.990384609870323 0.9999999988797315\n",
      "yahoo_A2 0.019230767289201187 0.004854368441889011 0.7818573730492826\n",
      "yahoo_A2 0.9999999485074653 0.985074619003914 0.999999998349235\n",
      "yahoo_A2 0.27639152381401827 0.2123890909622032 0.5998672984070941\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.9999999485074653 0.9850746190038153 0.9999999983492349\n",
      "yahoo_A2 0.9924811515178951 0.9842277467294286 0.9999055339624648\n",
      "yahoo_A2 0.9393938885725974 0.9763702978499592 0.9972756399084245\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746190043859 0.9999999983492349\n",
      "yahoo_A2 0.9999999490384642 0.9903846098898043 0.9999999988797315\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.29729726207314144 0.19956355868523945 0.7489524652298482\n",
      "yahoo_A2 0.9999999490384642 0.9903846098917293 0.9999999988797316\n",
      "yahoo_A2 0.0044247783176051365 0.0011086473390003108 0.02915766446454505\n",
      "yahoo_A2 0.9999999485074653 0.9850746187304231 0.999999998349235\n",
      "yahoo_A2 0.2482100020665201 0.0007923936731235576 -0.0\n",
      "yahoo_A2 0.4895104491564411 0.5001363532860037 0.7628706265226638\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840273\n",
      "yahoo_A2 0.4225351608311903 0.3908910784674091 0.7933182098796683\n",
      "yahoo_A2 0.9999999490384642 0.9903846098837732 0.9999999988797317\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.516853885058709 0.419758313796601 0.8977068755254651\n",
      "yahoo_A2 0.9951690311652572 0.9896947029538936 0.9998778987577682\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9852940662089126 0.9841864825083451 0.9999055339624646\n",
      "yahoo_A2 0.9658536076049996 0.9771974435278581 0.9970848584678569\n",
      "yahoo_A2 0.9999998500000123 0.0 0.999999899784027\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.9999999485074653 0.9850746190023966 0.999999998349235\n",
      "yahoo_A2 0.7389162054599757 0.7561845224479402 0.9531746021067916\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n",
      "yahoo_A2 0.17493470983952586 0.0364323502715013 0.1652700497949463\n",
      "yahoo_A2 0.9999999490384642 0.9903846098917741 0.9999999988797315\n",
      "yahoo_A2 0.005390835038978212 0.0013513512158509996 0.20410365125181545\n",
      "yahoo_A2 0.9999999485074653 0.9850746190042465 0.999999998349235\n",
      "yahoo_A2 0.9999999490384642 0.9903846098904413 0.9999999988797316\n",
      "yahoo_A2 0.9999998500000123 0.0 0.9999998997840271\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2318aa05d14c4ee493ef2c1496435e18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b0b77e54d16455fb4218f91713ec064",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/92 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A3 0.25396823169564314 0.0013262609223044285 -0.0\n",
      "yahoo_A3 0.442477841491114 0.16827980542465712 0.2879443964320134\n",
      "yahoo_A3 0.9011857202643405 0.9622200645924195 0.9837298247214965\n",
      "yahoo_A3 0.25396823169564314 0.004340662334214554 0.027039006318902215\n",
      "yahoo_A3 0.8251365620547075 0.8823212947085577 0.8662109219561183\n",
      "yahoo_A3 0.41726615387920135 0.043150605629332886 0.06153795737683343\n",
      "yahoo_A3 0.5852089616298454 0.16693101787221173 0.024729788263583928\n",
      "yahoo_A3 0.45614031550631234 0.06673494167946036 0.16091643990071816\n",
      "yahoo_A3 0.5210083646578659 0.36106625937459785 0.5205999086816473\n",
      "yahoo_A3 0.7134502462945892 0.5714902283497714 0.4704831719004913\n",
      "yahoo_A3 0.637770854195864 0.56669868732161 0.5638527705755241\n",
      "yahoo_A3 0.5783521396394755 0.4724997410194578 0.5637697484169977\n",
      "yahoo_A3 0.25825823385908653 0.12490565811617493 0.6132044812418928\n",
      "yahoo_A3 0.6766916843462069 0.22679362708249884 0.17874651309171286\n",
      "yahoo_A3 0.11563168070466742 0.0012106544339577385 0.0024213072633362588\n",
      "yahoo_A3 0.8181817681818212 0.9179499790026268 0.9332501666908628\n",
      "yahoo_A3 0.25396823169564314 0.005768119813341357 0.07716650131990542\n",
      "yahoo_A3 0.25396823169564314 0.09203049234831537 0.31224333900594897\n",
      "yahoo_A3 0.2812499757202169 0.02505895900829449 0.18059711089996855\n",
      "yahoo_A3 0.24701193044237577 0.0029849311837405893 0.045190433130609824\n",
      "yahoo_A3 0.2258064314906365 0.054654844897678787 0.16583930717666176\n",
      "yahoo_A3 0.45070419027970904 0.025837686120966288 0.14317668775023715\n",
      "yahoo_A3 0.6075948942156735 0.06910773306406771 0.18277598789462346\n",
      "yahoo_A3 0.4086798950024387 0.08965032741789997 0.8472857307719859\n",
      "yahoo_A3 0.25396823169564314 0.0013262609013137603 -0.0\n",
      "yahoo_A3 0.7961164541568512 0.8982550131609817 0.9464409404377719\n",
      "yahoo_A3 0.7228915157497496 0.711891180709674 0.8928106976609234\n",
      "yahoo_A3 0.25396823169564314 0.0013440869725268062 0.013297871007101737\n",
      "yahoo_A3 0.3763837331436143 0.014618235136716125 0.03359066437734164\n",
      "yahoo_A3 0.81034477870756 0.8680218711493342 0.8416044299904112\n",
      "yahoo_A3 0.8916876077927048 0.6271588518587361 0.3242780092924316\n",
      "yahoo_A3 0.7726638296349493 0.6583400376933547 0.5923864710416324\n",
      "yahoo_A3 0.5357736710795402 0.1239070957998471 0.03413689942770419\n",
      "yahoo_A3 0.7428570959510233 0.3211967145809075 0.3531351342455401\n",
      "yahoo_A3 0.6075948942156735 0.47388102501732954 0.6133242946068227\n",
      "yahoo_A3 0.7356321371911775 0.4708877903979453 0.4873614541747822\n",
      "yahoo_A3 0.7975077395910395 0.8217015933434961 0.78044122321854\n",
      "yahoo_A3 0.7035830158240435 0.7204517103626287 0.6450563344667073\n",
      "yahoo_A3 0.45070419027970904 0.2657508093102282 0.4901413846529178\n",
      "yahoo_A3 0.7978141594762488 0.8209754458372859 0.7201785181560871\n",
      "yahoo_A3 0.5306121945647692 0.4422946033961648 0.8497451796972182\n",
      "yahoo_A3 0.8762886093434505 0.9503437484476258 0.9450824124808477\n",
      "yahoo_A3 0.25396823169564314 0.0015479885925715634 0.1436170153529877\n",
      "yahoo_A3 0.323809496547848 0.02962523744352651 0.1863439369028965\n",
      "yahoo_A3 0.7153284209963267 0.4748790586889273 0.4384318459243932\n",
      "yahoo_A3 0.7743732113825961 0.38423905725959673 0.3117634841098844\n",
      "yahoo_A3 0.9128014339300801 0.9653799735806452 0.9331635313401956\n",
      "yahoo_A3 0.6846038411666818 0.5433330150294534 0.5718907319624753\n",
      "yahoo_A3 0.29906538349201256 0.15642165513618084 0.6273506009595924\n",
      "yahoo_A3 0.40579706895610423 0.24072799919648788 0.5545590925889269\n",
      "yahoo_A3 0.9056603267953037 0.9589519993510436 0.9869401615887086\n",
      "yahoo_A3 0.5874798941145886 0.005577221605851904 0.21498054052378263\n",
      "yahoo_A3 0.5259631101380745 0.1352431127883416 0.36895462656345196\n",
      "yahoo_A3 0.700147664776179 0.27702360787163977 0.35644547364387136\n",
      "yahoo_A3 0.6585365409949464 0.6647885610574145 0.7027512049701428\n",
      "yahoo_A3 0.9958847228437424 0.9915420196042039 0.9998968954532563\n",
      "yahoo_A3 0.6351350847630609 0.7396847244303264 0.7882238037376159\n",
      "yahoo_A3 0.3086419477537747 0.10138295647487663 0.6190613394252127\n",
      "yahoo_A3 0.7283236529018707 0.29567354680864383 0.3585399942553254\n",
      "yahoo_A3 0.6646433545110223 0.5880208903495583 0.655703408267887\n",
      "yahoo_A3 0.9578543554146325 0.9816040803705819 0.9953550670123292\n",
      "yahoo_A3 0.9142856640000028 0.8984063377668148 0.8790074875898558\n",
      "yahoo_A3 0.33333329394444905 0.22496436728436814 0.6351396264982487\n",
      "yahoo_A3 0.6943619722688439 0.5643253062833374 0.5575345362013421\n",
      "yahoo_A3 0.700147664776179 0.5599332884293124 0.49484286817045786\n",
      "yahoo_A3 0.7246376345305637 0.8244927742195404 0.7810111878833392\n",
      "yahoo_A3 0.442477841491114 0.2931006823193926 0.5402082758816391\n",
      "yahoo_A3 0.8541666175075981 0.9362002352068367 0.8329921169397438\n",
      "yahoo_A3 0.7301586835887245 0.7994134311999169 0.7647086944184094\n",
      "yahoo_A3 0.25049699588078084 0.0672154427256065 0.25858367062299853\n",
      "yahoo_A3 0.47346933740941805 0.39134442668447655 0.5666384244492302\n",
      "yahoo_A3 0.43694490352936993 0.10803680568281239 0.3954491833638726\n",
      "yahoo_A3 0.6531986027865676 0.7435953276307845 0.7806334151569099\n",
      "yahoo_A3 0.5395348356084412 0.5354698868415029 0.7755559560478159\n",
      "yahoo_A3 0.841269789947093 0.8832589525177542 0.9767692880406205\n",
      "yahoo_A3 0.25396823169564314 0.0014285723916390932 0.07180850494340174\n",
      "yahoo_A3 0.45070419027970904 0.09340669805634336 0.11850169290726635\n",
      "yahoo_A3 0.8533332825995091 0.9123828431030842 0.9596574535433053\n",
      "yahoo_A3 0.9481480968779176 0.9034541440495437 0.9120097694824041\n",
      "yahoo_A3 0.6865671188861692 0.40390148632211703 0.4211604445255748\n",
      "yahoo_A3 0.7949366597301744 0.8612891205390205 0.7971927684660567\n",
      "yahoo_A3 0.5783521396394755 0.138433072931404 0.16737003269469744\n",
      "yahoo_A3 0.6398763086823295 0.595297790803192 0.6030939757488188\n",
      "yahoo_A3 0.9877550512419851 0.9911180074958919 0.9996682308088657\n",
      "yahoo_A3 0.9816513252504023 0.9869361539706814 0.9983884285418985\n",
      "yahoo_A3 0.9820788023586567 0.9870514688460319 0.9945270762226152\n",
      "yahoo_A3 0.3301707502968744 0.03878148791689849 0.22110697965549903\n",
      "yahoo_A3 0.7811633870811333 0.4165013758965534 0.31310753503202465\n",
      "yahoo_A3 0.3882783568463308 0.012874735269554984 0.04574882630668321\n",
      "yahoo_A3 0.45070419027970904 0.1756412718043256 0.3565050240571419\n",
      "yahoo_A3 0.516020197625476 0.22445853410916297 0.46254017025142485\n",
      "yahoo_A3 0.8505746629967292 0.8408326366779788 0.9569359744636001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f2928180374428d82916e7125a212b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10fac2fef7204a4199a9c8bfc402af46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/82 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yahoo_A4 0.4853700147102329 0.10470647882008756 0.24286439642815996\n",
      "yahoo_A4 0.26890753963703323 0.01848030099177509 0.09576615316116278\n",
      "yahoo_A4 0.41726615387920135 0.03913256051502488 0.16893392005621644\n",
      "yahoo_A4 0.2945736181779962 0.0925633950576123 0.2143839275731135\n",
      "yahoo_A4 0.7182662043957131 0.81249375343508 0.8143517108462365\n",
      "yahoo_A4 0.7927272242988459 0.8549389222410892 0.9007693164164009\n",
      "yahoo_A4 0.6483516038220051 0.5961263517092834 0.6621561107801868\n",
      "yahoo_A4 0.40123452570302226 0.27660093009472175 0.5986772064115975\n",
      "yahoo_A4 0.5942491593197877 0.32925837818735 0.46298191934404526\n",
      "yahoo_A4 0.45070419027970904 0.19595777555348282 0.3627034433053316\n",
      "yahoo_A4 0.637770854195864 0.10853350132014533 0.0488832692671599\n",
      "yahoo_A4 0.3882783568463308 0.06552846791417004 0.2958134644533606\n",
      "yahoo_A4 0.45070419027970904 0.017635405848483887 0.13307184943254344\n",
      "yahoo_A4 0.3551401575753364 0.01026782732407385 0.0909691766701026\n",
      "yahoo_A4 0.07439824225732532 0.0011792458468121599 -0.0\n",
      "yahoo_A4 0.5151514716151449 0.3517887036358872 0.8463170923717859\n",
      "yahoo_A4 0.5942491593197877 0.3260447992376314 0.32118374555258594\n",
      "yahoo_A4 0.6075948942156735 0.13592342717057881 0.16868186490349638\n",
      "yahoo_A4 0.7675069552887851 0.48905024918015927 0.40321523903276424\n",
      "yahoo_A4 0.7465180556761697 0.829624900630715 0.8351143772265365\n",
      "yahoo_A4 0.25396823169564314 0.0030838535258143978 0.07948588817867687\n",
      "yahoo_A4 0.45070419027970904 0.09048123820022828 0.08382442712218492\n",
      "yahoo_A4 0.5714285304267189 0.5127404341197767 0.4879181303795873\n",
      "yahoo_A4 0.25396823169564314 0.009213687895913241 0.07015686656925826\n",
      "yahoo_A4 0.9999999492187527 0.9921874953675645 0.9999999988982372\n",
      "yahoo_A4 0.5454545056048113 0.05156114137658855 0.11913353220504115\n",
      "yahoo_A4 0.8328911978596936 0.8282934368302812 0.6100853194492863\n",
      "yahoo_A4 0.25396823169564314 0.0741755314940766 0.25049990632517544\n",
      "yahoo_A4 0.6334841151686119 0.6871785719959611 0.7443411030560062\n",
      "yahoo_A4 0.7726638296349493 0.5605522164420162 0.5079666593172376\n",
      "yahoo_A4 0.9999999452380977 0.9523809329644506 0.9999999949994318\n",
      "yahoo_A4 0.4086798950024387 0.16915028122009035 0.40010555928787983\n",
      "yahoo_A4 0.25396823169564314 0.022931231521958487 0.22228404455825945\n",
      "yahoo_A4 0.42293903459873594 0.10997841156795907 0.33780680188358875\n",
      "yahoo_A4 0.6666666220202049 0.639522458413786 0.5915576043833205\n",
      "yahoo_A4 0.6279999543696031 0.37150562742962867 0.5362441922399821\n",
      "yahoo_A4 0.6891891387805916 0.726573499739297 0.7160086815528806\n",
      "yahoo_A4 0.6075948942156735 0.4904790072054527 0.5406728350690201\n",
      "yahoo_A4 0.4756944080240913 0.1307662658557272 0.32769755272665474\n",
      "yahoo_A4 0.4074073742112509 0.352162662094179 0.4245828000085905\n",
      "yahoo_A4 0.7290969400163343 0.766956243764442 0.8142814340191844\n",
      "yahoo_A4 0.03131990741758405 0.002371979964699273 0.277136235224682\n",
      "yahoo_A4 0.5806451198959446 0.27667607708522113 0.5080777094450833\n",
      "yahoo_A4 0.3011582754610119 0.07247374406139044 0.2379200849106646\n",
      "yahoo_A4 0.3703703400548721 0.07450274773774858 0.30899109573122824\n",
      "yahoo_A4 0.6776858994604229 0.6643544630136944 0.9266058117056625\n",
      "yahoo_A4 0.6666666215277807 0.5993699211548935 0.6220568124259738\n",
      "yahoo_A4 0.6543209408779183 0.7059479982482864 0.9443787209147582\n",
      "yahoo_A4 0.46422334988015346 0.06990944496433969 0.04962201903628317\n",
      "yahoo_A4 0.45070419027970904 0.20502606006509855 0.40623688711657874\n",
      "yahoo_A4 0.8505746632358332 0.9019875464208157 0.8395251478008684\n",
      "yahoo_A4 0.7522935275594118 0.8249396737806217 0.8566491301997148\n",
      "yahoo_A4 0.25396823169564314 0.02211488565272913 0.3273504741619639\n",
      "yahoo_A4 0.9999999484375026 0.9843749918253891 0.9999999981715425\n",
      "yahoo_A4 0.6217008345095104 0.37938661969498144 0.6712374943431416\n",
      "yahoo_A4 0.6256409756949415 0.7081908138935786 0.8636365032591038\n",
      "yahoo_A4 0.5045871061063932 0.4939653020041795 0.590117983541071\n",
      "yahoo_A4 0.2984495868960721 0.047288830729034795 0.27833276349815056\n",
      "yahoo_A4 0.7424241913452743 0.7027016509823147 0.9576961418659147\n",
      "yahoo_A4 0.5919999581337628 0.02248597809409194 0.07137013894507956\n",
      "yahoo_A4 0.45070419027970904 0.09367126121938826 0.11774049216619302\n",
      "yahoo_A4 0.6982248064178453 0.3039507459269933 0.22049174131251148\n",
      "yahoo_A4 0.5806451198959446 0.39589857746548907 0.6481986356460757\n",
      "yahoo_A4 0.7003154081660714 0.7667737112258373 0.8273091552597571\n",
      "yahoo_A4 0.25396823169564314 0.002834784297811184 0.07025806774643863\n",
      "yahoo_A4 0.4957264582716078 0.2833631664821113 0.487561131236475\n",
      "yahoo_A4 0.9999999491869945 0.9918699139139096 0.9999999988715345\n",
      "yahoo_A4 0.2544731387231303 0.07708266938634693 0.09689650883384525\n",
      "yahoo_A4 0.5919999581337628 0.4363660969911196 0.5478521823249838\n",
      "yahoo_A4 0.6544342065351805 0.6891609816113528 0.6864661809063485\n",
      "yahoo_A4 0.25049699588078084 0.0054469487617334455 0.13191454600145033\n",
      "yahoo_A4 0.7535410293108873 0.7280715731578133 0.6248836577956786\n",
      "yahoo_A4 0.5897435479700884 0.12325119309713714 0.249799470950654\n",
      "yahoo_A4 0.6544342065351805 0.3228970982137938 0.3714372660624779\n",
      "yahoo_A4 0.45070419027970904 0.22693613370103338 0.4910354894753449\n",
      "yahoo_A4 0.6075948942156735 0.33564775970030764 0.4670313373264791\n",
      "yahoo_A4 0.45070419027970904 0.05303700794643538 0.08537789312964424\n",
      "yahoo_A4 0.35205989595309467 0.0954529081808087 0.27691471759992126\n",
      "yahoo_A4 0.45070419027970904 0.14703244030365806 0.2807778300546092\n",
      "yahoo_A4 0.8490027989610504 0.736034907670212 0.8860100572823665\n",
      "yahoo_A4 0.6419752648529217 0.12630079622947696 0.1153750613761058\n",
      "yahoo_A4 0.2608695424299727 0.0013333343000693242 -0.0\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_yahoo_A1, load_yahoo_A2, load_yahoo_A3, load_yahoo_A4]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = CNN_VAE(X_train)\n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "    \n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bd9e37fd-124c-4f5c-8f1f-a9d61a4b0315",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>yahoo_A1</th>\n",
       "      <td>0.862802</td>\n",
       "      <td>0.785256</td>\n",
       "      <td>0.876318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A2</th>\n",
       "      <td>0.681964</td>\n",
       "      <td>0.469883</td>\n",
       "      <td>0.799709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A3</th>\n",
       "      <td>0.597183</td>\n",
       "      <td>0.435267</td>\n",
       "      <td>0.513349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yahoo_A4</th>\n",
       "      <td>0.542656</td>\n",
       "      <td>0.352417</td>\n",
       "      <td>0.450631</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                f1    pr_auc   roc_auc\n",
       "dataset                               \n",
       "yahoo_A1  0.862802  0.785256  0.876318\n",
       "yahoo_A2  0.681964  0.469883  0.799709\n",
       "yahoo_A3  0.597183  0.435267  0.513349\n",
       "yahoo_A4  0.542656  0.352417  0.450631"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yahoo_results = pd.DataFrame(total_scores)\n",
    "yahoo_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd5b7099-3ad2-4187-8afe-c82f79098d89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8890a678-6116-4222-b0ff-395fcd7f41aa",
   "metadata": {},
   "source": [
    "### NASA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e4387a80-585a-488e-94a8-546f42d47632",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "106a238d-c5cb-49e9-9764-b1bc37f09436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd6af2e64fdc4ce0ab2b1dd28e8648e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a59011ff09847268042e8d8a4662cdd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D1 0.25300017462469454 0.13296894487001693 0.37454971742462734\n",
      "D2 0.2699530282282525 0.21756620717794117 0.5025344006309231\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_nasa]:\n",
    "    datasets = loader(100, 100)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "        \n",
    "        model = CNN_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "        \n",
    "        total_scores['dataset'].append(f'D{i+1}')\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(f'D{i+1}', np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "abb7a6a5-3924-4a33-bc54-871559819602",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1</th>\n",
       "      <td>0.253000</td>\n",
       "      <td>0.132969</td>\n",
       "      <td>0.374550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2</th>\n",
       "      <td>0.269953</td>\n",
       "      <td>0.217566</td>\n",
       "      <td>0.502534</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "D1       0.253000  0.132969  0.374550\n",
       "D2       0.269953  0.217566  0.502534"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nasa_results = pd.DataFrame(total_scores)\n",
    "nasa_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c85311b3-f660-404e-9f37-6bf254f93d14",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "41754734-5c75-4caf-9069-fb3b39c7a0da",
   "metadata": {},
   "source": [
    "### SMD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "60a26807-8359-42d4-82b4-9e1d37b6acaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "41af318e-1b08-4cc9-a680-6b4a978544f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "611d7444782c43daa50351535ae87c20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a0221c42db24a4dba92e7bdf42081e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smd 0.20232181144029696 0.5753855920741581 0.521888680406877\n",
      "smd 0.7141517779711564 0.5516386577636502 0.9182976553226475\n",
      "smd 0.20111467783211762 0.1745763678070538 0.5911903376062385\n",
      "smd 0.25709511124931544 0.2873047363563944 0.5907813393239671\n",
      "smd 0.689655126355723 0.7511517867465145 0.9516453143237289\n",
      "smd 0.7267976360915273 0.8032251567456201 0.8869642779430862\n",
      "smd 0.4359626367927769 0.4486698601779356 0.7034005023443057\n",
      "smd 0.30716719136819937 0.30611850236707355 0.735342488501898\n",
      "smd 0.5271035125639076 0.5747599561245318 0.8625485716445264\n",
      "smd 0.2596273980668859 0.30792785511488585 0.5433837534356515\n",
      "smd 0.710753256473802 0.7487777642568118 0.840550096292183\n",
      "smd 0.24491919329678966 0.27030654229683954 0.5713197092700563\n",
      "smd 0.5960077243623663 0.6136700800268254 0.9084684127032229\n",
      "smd 0.5246179575739174 0.5269636196020161 0.711061941376982\n",
      "smd 0.5490013458230265 0.5981668008586271 0.9217290815524782\n",
      "smd 0.967741885076135 0.9561753256936034 0.9814953942223376\n",
      "smd 0.7643821429023572 0.7830142326746439 0.8193630602023578\n",
      "smd 0.5652173505007192 0.699475612131001 0.69730680426252\n",
      "smd 0.5449794276354869 0.5043435290990623 0.7883700465100326\n",
      "smd 0.7834710264654082 0.7471532225898483 0.9487515136910479\n",
      "smd 0.24621589436610591 0.1724703459307044 0.6570616868611716\n",
      "smd 0.4021589225949868 0.24521510182242995 0.8147318039072887\n",
      "smd 0.7722695095049712 0.6898421424208484 0.8748891093613916\n",
      "smd 0.37123535935831353 0.28005437240456565 0.6975781320688875\n",
      "smd 0.8176936345205004 0.8433142673482618 0.985612526989002\n",
      "smd 0.21666664729563664 0.2825547768821436 0.6788895231448112\n",
      "smd 0.47643307816802505 0.5074327082263319 0.8697230047119835\n",
      "smd 0.5603112435676573 0.45179899186890904 0.695286608471308\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_smd]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "\n",
    "        model = CNN_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "23474773-bac1-4145-ac75-c1a686bca5c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>smd</th>\n",
       "      <td>0.515538</td>\n",
       "      <td>0.525053</td>\n",
       "      <td>0.777415</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "smd      0.515538  0.525053  0.777415"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smd_results = pd.DataFrame(total_scores)\n",
    "smd_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9c021ae-607d-4bfa-9330-cb8c56b299cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "89bf290c-6263-49f8-b544-6537bcfeaeb8",
   "metadata": {},
   "source": [
    "### ECG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d2a5ab3b-f911-4698-8224-5bad41d0d2f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d09ecce4-8928-407a-ac46-085b003de37f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c983672e7f7e4b10a300ab8e702ad179",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e31705947d0468497f652de5afdd818",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D1 0.5599999552000028 0.4879766675912788 0.6531181873780253\n",
      "D2 0.6315788941828289 0.526206208538601 0.8413952868087482\n",
      "D3 0.16666664861111138 0.02883169106815893 0.3237835168736265\n",
      "D4 0.3809523428571462 0.20399695917390048 0.6491525347516718\n",
      "WARNING:tensorflow:5 out of the last 17 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9d8ce98dd0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "D5 0.5333332833333373 0.30952770957349535 0.8406759818822656\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9db8688290> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "D6 0.31578942700831664 0.1520637224813504 0.6756206698865388\n",
      "D7 0.05374279705276701 0.017278843643561987 0.434757307534348\n",
      "D8 0.19251335137979503 0.08028738253335913 0.41571467721684796\n",
      "D9 0.38805966963689265 0.16173260225718553 0.34468183054611445\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_ecg]:\n",
    "    datasets = loader(32, 16)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "\n",
    "        model = CNN_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(f'D{i+1}')\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(f'D{i+1}', np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "427055ff-a5d6-4994-96a1-d55878130385",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1</th>\n",
       "      <td>0.560000</td>\n",
       "      <td>0.487977</td>\n",
       "      <td>0.653118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2</th>\n",
       "      <td>0.631579</td>\n",
       "      <td>0.526206</td>\n",
       "      <td>0.841395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D3</th>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.028832</td>\n",
       "      <td>0.323784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D4</th>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.203997</td>\n",
       "      <td>0.649153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D5</th>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.309528</td>\n",
       "      <td>0.840676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D6</th>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.152064</td>\n",
       "      <td>0.675621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D7</th>\n",
       "      <td>0.053743</td>\n",
       "      <td>0.017279</td>\n",
       "      <td>0.434757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D8</th>\n",
       "      <td>0.192513</td>\n",
       "      <td>0.080287</td>\n",
       "      <td>0.415715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D9</th>\n",
       "      <td>0.388060</td>\n",
       "      <td>0.161733</td>\n",
       "      <td>0.344682</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc   roc_auc\n",
       "dataset                              \n",
       "D1       0.560000  0.487977  0.653118\n",
       "D2       0.631579  0.526206  0.841395\n",
       "D3       0.166667  0.028832  0.323784\n",
       "D4       0.380952  0.203997  0.649153\n",
       "D5       0.533333  0.309528  0.840676\n",
       "D6       0.315789  0.152064  0.675621\n",
       "D7       0.053743  0.017279  0.434757\n",
       "D8       0.192513  0.080287  0.415715\n",
       "D9       0.388060  0.161733  0.344682"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ecg_results = pd.DataFrame(total_scores)\n",
    "ecg_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55c21ccb-6ee6-4ea4-a5b8-2aa619fd9b03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "89524a09-75d8-4b7b-8734-851c2c63c710",
   "metadata": {},
   "source": [
    "### Power Demand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "601f6885-93dc-443e-b085-bd6bc36882d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b4b978e2-8105-4622-bb2c-abbb1fddd5b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f04a17eaa644b3fb94b1394ffba02de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "power_demand 0.3606206231571944 0.13386993729632082 0.33812430414059097\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_power_demand]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "\n",
    "        model = CNN_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b325edbc-9c9c-429e-a8ab-78353b6cdadc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>power_demand</th>\n",
       "      <td>0.360621</td>\n",
       "      <td>0.13387</td>\n",
       "      <td>0.338124</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    f1   pr_auc   roc_auc\n",
       "dataset                                  \n",
       "power_demand  0.360621  0.13387  0.338124"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power_results = pd.DataFrame(total_scores)\n",
    "power_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b68c397a-6190-470b-b964-6de81289a898",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9d780512-bc48-49b4-8ee7-31e4e418cd7e",
   "metadata": {},
   "source": [
    "### 2D Gesture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2587ede7-1e7f-435e-b382-29388c3b2d76",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_scores = {'dataset': [], 'f1': [], 'pr_auc': [], 'roc_auc': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "597108eb-cc0f-450c-9301-9144e9b52fdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d6a9e9943d948f5b468e19513355f21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gesture 0.45591589948213174 0.3357151688266453 0.6040000597857351\n"
     ]
    }
   ],
   "source": [
    "for loader in [load_gesture]:\n",
    "    datasets = loader(64, 1)\n",
    "    x_trains, x_tests, y_tests = datasets['x_train'], datasets['x_test'], datasets['y_test']\n",
    "    \n",
    "    for i in tqdm(range(len(x_trains))):\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "        X_train = x_trains[i]\n",
    "        X_test = x_tests[i]\n",
    "\n",
    "        model = CNN_VAE(X_train)\n",
    "        \n",
    "        X_test_rec = model.decoder.predict(model.encoder.predict(X_test)[-1])\n",
    "        scores = evaluate(X_test, X_test_rec, y_tests[i], is_reconstructed=True)\n",
    "\n",
    "        total_scores['dataset'].append(loader.__name__.replace('load_', ''))\n",
    "        total_scores['f1'].append(np.max(scores['f1']))\n",
    "        total_scores['pr_auc'].append(scores['pr_auc'])\n",
    "        total_scores['roc_auc'].append(scores['roc_auc'])\n",
    "        print(loader.__name__.replace('load_', ''), np.max(scores['f1']), scores['pr_auc'], scores['roc_auc'])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ce3433e8-bb01-4ca0-b861-43676a8aa3dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>pr_auc</th>\n",
       "      <th>roc_auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gesture</th>\n",
       "      <td>0.455916</td>\n",
       "      <td>0.335715</td>\n",
       "      <td>0.604</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               f1    pr_auc  roc_auc\n",
       "dataset                             \n",
       "gesture  0.455916  0.335715    0.604"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gesture_results = pd.DataFrame(total_scores)\n",
    "gesture_results.groupby('dataset').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6b4d96d-0369-4d64-9257-824070ace4c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}