Classification_on_CNN_VAE_latent_space_with_NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SCOGgPDrM9zN"
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import time\n",
    "\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F \n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import OrderedDict\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "\n",
    "from utils.inference import Trainer, plot_loss\n",
    "from utils.models import DNN\n",
    "import utils.datasets as d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1Z5N6d8DNI-z"
   },
   "source": [
    "# Without sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "voq6CalmM92W"
   },
   "outputs": [],
   "source": [
    "X_train = np.load(\"data/encoded_data_train_CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300.npy\")\n",
    "y_train = np.load(\"data/labels_train.npy\")\n",
    "X_test = np.load(\"data/encoded_data_test_CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300.npy\")\n",
    "y_test = np.load(\"data/labels_test.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tu-V4WzXM96g"
   },
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NBVcA97PM99W"
   },
   "outputs": [],
   "source": [
    "train = d.H_alphaSequences(X_train, y_train)\n",
    "valid = d.H_alphaSequences(X_valid, y_valid)\n",
    "test = d.H_alphaSequences(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8M3gLPnxM-Aq"
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset = train, batch_size=512, shuffle=True)\n",
    "valid_loader = torch.utils.data.DataLoader(dataset = valid, batch_size=512, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2m2EKociNVJg"
   },
   "outputs": [],
   "source": [
    "DNN1 = nn.Sequential(\n",
    "            nn.Linear(in_features=15, out_features=64),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(in_features=64, out_features=128),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(in_features=128, out_features=4),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QocCCcgSNVMP"
   },
   "outputs": [],
   "source": [
    "optimizer= torch.optim.Adam(DNN1.parameters(), lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "z0HvwkdgNVV8",
    "outputId": "f8fd8d7a-0ea5-46a5-da05-0c03b99faf3f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "m1 = Trainer(\n",
    "        model=DNN1,\n",
    "        optimizer=optimizer,\n",
    "        loss_function=nn.CrossEntropyLoss(),\n",
    "        scheduler=torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30], gamma=0.1),\n",
    "        tensorboard=True,\n",
    "        model_name=\"DNN_on_CNN-VAE_latent_space_m.IV_lr-{1e-2,1e-3}_epochs-512\",\n",
    "        verbose=True\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Av602ZNxNVcl",
    "outputId": "97a3e384-0b44-4024-bca9-50104a6c3ae5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], average_loss:0.7551, validation_loss:0.6017, val_accuracy:0.7662\n",
      "Epoch [2/100], average_loss:0.5648, validation_loss:0.5291, val_accuracy:0.8312\n",
      "Epoch [3/100], average_loss:0.5102, validation_loss:0.4963, val_accuracy:0.7662\n",
      "Epoch [4/100], average_loss:0.4727, validation_loss:0.4679, val_accuracy:0.7273\n",
      "Epoch [5/100], average_loss:0.4417, validation_loss:0.4357, val_accuracy:0.8312\n",
      "Epoch [6/100], average_loss:0.4232, validation_loss:0.4357, val_accuracy:0.7792\n",
      "Epoch [7/100], average_loss:0.4155, validation_loss:0.4135, val_accuracy:0.8442\n",
      "Epoch [8/100], average_loss:0.4067, validation_loss:0.4092, val_accuracy:0.8571\n",
      "Epoch [9/100], average_loss:0.4009, validation_loss:0.4131, val_accuracy:0.8442\n",
      "Epoch [10/100], average_loss:0.3968, validation_loss:0.4144, val_accuracy:0.8312\n",
      "Epoch [11/100], average_loss:0.3898, validation_loss:0.3971, val_accuracy:0.8831\n",
      "Epoch [12/100], average_loss:0.3813, validation_loss:0.3975, val_accuracy:0.8701\n",
      "Epoch [13/100], average_loss:0.3827, validation_loss:0.3949, val_accuracy:0.8442\n",
      "Epoch [14/100], average_loss:0.3754, validation_loss:0.3870, val_accuracy:0.8831\n",
      "Epoch [15/100], average_loss:0.3697, validation_loss:0.3879, val_accuracy:0.8312\n",
      "Epoch [16/100], average_loss:0.3642, validation_loss:0.3843, val_accuracy:0.8312\n",
      "Epoch [17/100], average_loss:0.3629, validation_loss:0.3798, val_accuracy:0.8701\n",
      "Epoch [18/100], average_loss:0.3559, validation_loss:0.3809, val_accuracy:0.8571\n",
      "Epoch [19/100], average_loss:0.3535, validation_loss:0.3841, val_accuracy:0.7922\n",
      "Epoch [20/100], average_loss:0.3480, validation_loss:0.3817, val_accuracy:0.8052\n",
      "Epoch [21/100], average_loss:0.3461, validation_loss:0.3730, val_accuracy:0.8052\n",
      "Epoch [22/100], average_loss:0.3411, validation_loss:0.3702, val_accuracy:0.8442\n",
      "Epoch [23/100], average_loss:0.3394, validation_loss:0.3665, val_accuracy:0.8831\n",
      "Epoch [24/100], average_loss:0.3370, validation_loss:0.3817, val_accuracy:0.9221\n",
      "Epoch [25/100], average_loss:0.3359, validation_loss:0.3680, val_accuracy:0.8182\n",
      "Epoch [26/100], average_loss:0.3325, validation_loss:0.3608, val_accuracy:0.8831\n",
      "Epoch [27/100], average_loss:0.3307, validation_loss:0.3813, val_accuracy:0.7532\n",
      "Epoch [28/100], average_loss:0.3286, validation_loss:0.3649, val_accuracy:0.8052\n",
      "Epoch [29/100], average_loss:0.3297, validation_loss:0.3691, val_accuracy:0.8312\n",
      "Epoch [30/100], average_loss:0.3248, validation_loss:0.3671, val_accuracy:0.8961\n",
      "Epoch [31/100], average_loss:0.3123, validation_loss:0.3605, val_accuracy:0.8182\n",
      "Epoch [32/100], average_loss:0.3098, validation_loss:0.3599, val_accuracy:0.8312\n",
      "Epoch [33/100], average_loss:0.3094, validation_loss:0.3608, val_accuracy:0.7792\n",
      "Epoch [34/100], average_loss:0.3095, validation_loss:0.3557, val_accuracy:0.8312\n",
      "Epoch [35/100], average_loss:0.3084, validation_loss:0.3574, val_accuracy:0.8182\n",
      "Epoch [36/100], average_loss:0.3085, validation_loss:0.3566, val_accuracy:0.8312\n",
      "Epoch [37/100], average_loss:0.3079, validation_loss:0.3596, val_accuracy:0.8442\n",
      "Epoch [38/100], average_loss:0.3080, validation_loss:0.3600, val_accuracy:0.7792\n",
      "Epoch [39/100], average_loss:0.3078, validation_loss:0.3534, val_accuracy:0.8831\n",
      "Epoch [40/100], average_loss:0.3081, validation_loss:0.3617, val_accuracy:0.7792\n",
      "Epoch [41/100], average_loss:0.3075, validation_loss:0.3600, val_accuracy:0.8442\n",
      "Epoch [42/100], average_loss:0.3069, validation_loss:0.3622, val_accuracy:0.7013\n",
      "Epoch [43/100], average_loss:0.3068, validation_loss:0.3563, val_accuracy:0.8442\n",
      "Epoch [44/100], average_loss:0.3064, validation_loss:0.3505, val_accuracy:0.9481\n",
      "Epoch [45/100], average_loss:0.3062, validation_loss:0.3530, val_accuracy:0.8442\n",
      "Epoch [46/100], average_loss:0.3062, validation_loss:0.3629, val_accuracy:0.7662\n",
      "Epoch [47/100], average_loss:0.3057, validation_loss:0.3565, val_accuracy:0.8182\n",
      "Epoch [48/100], average_loss:0.3059, validation_loss:0.3604, val_accuracy:0.8182\n",
      "Epoch [49/100], average_loss:0.3051, validation_loss:0.3534, val_accuracy:0.9091\n",
      "Epoch [50/100], average_loss:0.3049, validation_loss:0.3648, val_accuracy:0.7922\n",
      "Epoch [51/100], average_loss:0.3047, validation_loss:0.3550, val_accuracy:0.8571\n",
      "Epoch [52/100], average_loss:0.3049, validation_loss:0.3586, val_accuracy:0.8182\n",
      "Epoch [53/100], average_loss:0.3041, validation_loss:0.3572, val_accuracy:0.8312\n",
      "Epoch [54/100], average_loss:0.3040, validation_loss:0.3554, val_accuracy:0.9091\n",
      "Epoch [55/100], average_loss:0.3041, validation_loss:0.3549, val_accuracy:0.8571\n",
      "Epoch [56/100], average_loss:0.3032, validation_loss:0.3568, val_accuracy:0.9091\n",
      "Epoch [57/100], average_loss:0.3028, validation_loss:0.3593, val_accuracy:0.8312\n",
      "Epoch [58/100], average_loss:0.3029, validation_loss:0.3561, val_accuracy:0.8312\n",
      "Epoch [59/100], average_loss:0.3027, validation_loss:0.3562, val_accuracy:0.8701\n",
      "Epoch [60/100], average_loss:0.3026, validation_loss:0.3567, val_accuracy:0.8312\n",
      "Epoch [61/100], average_loss:0.3027, validation_loss:0.3547, val_accuracy:0.8701\n",
      "Epoch [62/100], average_loss:0.3023, validation_loss:0.3577, val_accuracy:0.8961\n",
      "Epoch [63/100], average_loss:0.3020, validation_loss:0.3542, val_accuracy:0.9221\n",
      "Epoch [64/100], average_loss:0.3016, validation_loss:0.3514, val_accuracy:0.8831\n",
      "Epoch [65/100], average_loss:0.3010, validation_loss:0.3566, val_accuracy:0.8701\n",
      "Epoch [66/100], average_loss:0.3011, validation_loss:0.3533, val_accuracy:0.8961\n",
      "Epoch [67/100], average_loss:0.3007, validation_loss:0.3557, val_accuracy:0.9091\n",
      "Epoch [68/100], average_loss:0.3006, validation_loss:0.3576, val_accuracy:0.8961\n",
      "Epoch [69/100], average_loss:0.3002, validation_loss:0.3592, val_accuracy:0.8571\n",
      "Epoch [70/100], average_loss:0.3001, validation_loss:0.3550, val_accuracy:0.8701\n",
      "Epoch [71/100], average_loss:0.3001, validation_loss:0.3561, val_accuracy:0.8701\n",
      "Epoch [72/100], average_loss:0.3000, validation_loss:0.3589, val_accuracy:0.8182\n",
      "Epoch [73/100], average_loss:0.2995, validation_loss:0.3586, val_accuracy:0.7922\n",
      "Epoch [74/100], average_loss:0.2991, validation_loss:0.3574, val_accuracy:0.8312\n",
      "Epoch [75/100], average_loss:0.2990, validation_loss:0.3632, val_accuracy:0.7532\n",
      "Epoch [76/100], average_loss:0.2991, validation_loss:0.3562, val_accuracy:0.8442\n",
      "Epoch [77/100], average_loss:0.2983, validation_loss:0.3594, val_accuracy:0.7662\n",
      "Epoch [78/100], average_loss:0.2980, validation_loss:0.3587, val_accuracy:0.8831\n",
      "Epoch [79/100], average_loss:0.2979, validation_loss:0.3501, val_accuracy:0.9481\n",
      "Epoch [80/100], average_loss:0.2977, validation_loss:0.3581, val_accuracy:0.7403\n",
      "Epoch [81/100], average_loss:0.2977, validation_loss:0.3524, val_accuracy:0.8701\n",
      "Epoch [82/100], average_loss:0.2971, validation_loss:0.3551, val_accuracy:0.8571\n",
      "Epoch [83/100], average_loss:0.2974, validation_loss:0.3569, val_accuracy:0.8312\n",
      "Epoch [84/100], average_loss:0.2970, validation_loss:0.3607, val_accuracy:0.8442\n",
      "Epoch [85/100], average_loss:0.2965, validation_loss:0.3581, val_accuracy:0.8442\n",
      "Epoch [86/100], average_loss:0.2965, validation_loss:0.3569, val_accuracy:0.8052\n",
      "Epoch [87/100], average_loss:0.2965, validation_loss:0.3570, val_accuracy:0.9091\n",
      "Epoch [88/100], average_loss:0.2961, validation_loss:0.3545, val_accuracy:0.8571\n",
      "Epoch [89/100], average_loss:0.2956, validation_loss:0.3584, val_accuracy:0.8182\n",
      "Epoch [90/100], average_loss:0.2956, validation_loss:0.3603, val_accuracy:0.7792\n",
      "Epoch [91/100], average_loss:0.2954, validation_loss:0.3526, val_accuracy:0.9091\n",
      "Epoch [92/100], average_loss:0.2951, validation_loss:0.3542, val_accuracy:0.8312\n",
      "Epoch [93/100], average_loss:0.2951, validation_loss:0.3603, val_accuracy:0.7662\n",
      "Epoch [94/100], average_loss:0.2951, validation_loss:0.3570, val_accuracy:0.8182\n",
      "Epoch [95/100], average_loss:0.2944, validation_loss:0.3559, val_accuracy:0.8312\n",
      "Epoch [96/100], average_loss:0.2941, validation_loss:0.3581, val_accuracy:0.8312\n",
      "Epoch [97/100], average_loss:0.2938, validation_loss:0.3526, val_accuracy:0.8831\n",
      "Epoch [98/100], average_loss:0.2939, validation_loss:0.3540, val_accuracy:0.8831\n",
      "Epoch [99/100], average_loss:0.2939, validation_loss:0.3564, val_accuracy:0.8701\n",
      "Epoch [100/100], average_loss:0.2932, validation_loss:0.3570, val_accuracy:0.8182\n"
     ]
    }
   ],
   "source": [
    "lh = m1(epochs=100, train_loader=train_loader, validation_loader=valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8aB-SIR8NVaj"
   },
   "outputs": [],
   "source": [
    "m1.tb.close()\n",
    "\n",
    "m1.model.eval()\n",
    "with torch.no_grad():\n",
    "    x_pred = np.argmax(m1.model(torch.tensor(X_test).float().to(m1.device)).cpu().detach(), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "IYCLgA58NVQ5",
    "outputId": "04c01405-e732-4fdd-953f-ac866c0a11cc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 0, 1,  ..., 3, 1, 0])"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "64oUK08XNVPE",
    "outputId": "77af5ad0-2ccb-40c3-e8c2-c5cdd820fb71"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8292950992661148"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred=x_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "KzNmYtFxOgS_",
    "outputId": "defdd07f-72b8-4c2b-e6aa-46a0a884d706"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8604804015296367"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "SodFtzG_OgWG",
    "outputId": "3f863dfc-29e0-4304-b201-67413c3a0674"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3095,  998,   28,  114],\n",
       "       [ 626, 9675,   30,  114],\n",
       "       [  19,   44,  401,   15],\n",
       "       [ 204,  132,   11, 1230]])"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 442
    },
    "colab_type": "code",
    "id": "TanFgK7OOgay",
    "outputId": "d3ef996a-7bc7-4dde-ed9a-9663e9465f0f"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAGpCAYAAADFpuEPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3hcV5038O+Zqt57s2RZki33WLGd\n7jTHCSmEhZCw1BcILQu7lN2EkmUDS1heYN8FsrDZhYeyJCGUQBJM7JCeOHEsx12yious3vtIU8/7\nx5HkkXSnSXN9R8r38zx67Lkzc+dIGs1855TfEVJKEBEREdGFZTK6AURERERvRwxhRERERAZgCCMi\nIiIyAEMYERERkQEYwoiIiIgMYDG6AZHKysqSpaWlRjeDiIiIKKSDBw/2SSmzta5bciGstLQUtbW1\nRjeDiIiIKCQhREug6zgcSURERGQAhjAiIiIiAzCEERERERmAIYyIiIjIAAxhRERERAZgCCMiIiIy\nAEMYERERkQEYwoiIiIgMwBBGREREZACGMCIiIiIDMIQRERERGYAhjIiIiMgAuoUwIcTPhBA9Qojj\nAa4XQogfCCGahRBHhRAX6dUWIiIiolijZ0/YzwHsCnL9jQAqpr7uBvBjHdtCREREFFN0C2FSypcB\nDAS5yW0AfimVNwCkCSHy9WoPERERUSwxck5YIYBWv8ttU8fmEULcLYSoFULU9vb26t6wp4506P4Y\nRERE9Pa2JCbmSykfllLWSClrsrOzdX+85p4x3R+DiIiI3t6MDGHtAIr9LhdNHTOcNLoBREREtOwZ\nGcKeBPDBqVWS2wEMSyk7DWwPERER0QVj0evEQohHAewAkCWEaAPwzwCsACCl/AmA3QBuAtAMwAHg\nI3q1JVLC6AYQERHRsqdbCJNS3hXiegngM3o9PhEREVEsWxIT84mIiIiWG4YwIiIiIgMwhBEREREZ\ngCGMiIiIyAAMYRokALVugIiIiEgfDGEabGYBt5chjIiIiPTDEKbBbjHD5fUZ3QwiIiJaxhjCNNit\nJjjdXqObQURERMsYQ5gGu8UEp4c9YURERKQfhjANdouZIYyIiIh0xRCmQfWEcTiSiIiI9MMQpkHN\nCWNPGBEREemHIUyDzczhSCIiItIXQ5gGu5XDkURERKQvhjANdosJLvaEERERkY4YwjRwdSQRERHp\njSFMA1dHEhERkd4YwjRwdSQRERHpjSFMA4cjiYiISG8MYRo4HElERER6YwjTYLNwOJKIiIj0xRCm\nwWIScPuk0c0gIiKiZYwhTIMQAsLoRhAREdGyxhBGREREZACGMCIiIiIDMIQRERERGYAhjIiIiMgA\nDGFEREREBmAIC4AFKoiIiEhPDGFEREREBmAIC4B1woiIiEhPDGFEREREBmAIIyIiIjIAQxgRERGR\nARjCiIiIiAzAEEZERERkAF1DmBBilxCiQQjRLIS4V+P6FUKI54QQR4UQLwohivRsTyRYJ4yIiIj0\npFsIE0KYATwE4EYA1QDuEkJUz7nZdwH8Ukq5AcADAB7Uqz2RMgnA52MUIyIiIn3o2RO2FUCzlPK0\nlNIF4DEAt825TTWA56f+/4LG9YaxW8xweX1GN4OIiIiWKT1DWCGAVr/LbVPH/B0B8K6p/98OIFkI\nkTn3REKIu4UQtUKI2t7eXl0aO5fdYoLTzRBGRERE+jB6Yv4XAVwlhDgE4CoA7QC8c28kpXxYSlkj\npazJzs6+IA2zW01weuY1hYiIiCgqLDqeux1Asd/loqljM6SUHZjqCRNCJAH4GynlkI5tCpvdYobT\nw54wIiIi0oeePWEHAFQIIcqEEDYAdwJ40v8GQogsIcR0G+4D8DMd2xMRu4U9YURERKQf3UKYlNID\n4B4AewDUA3hcSnlCCPGAEOLWqZvtANAghGgEkAvgX/VqT6TsFhMmOSeMiIiIdKLncCSklLsB7J5z\n7H6///8OwO/0bMNC2SwmDkcSERGRboyemB+z1JwwDkcSERGRPhjCAlCrI9kTRkRERPpgCAvAbjHB\nxRBGREREOmEIC4AlKoiIiEhPDGEBqIr5nBNGRERE+mAIC4BzwoiIiEhPDGEBcDiSiIiI9MQQFgAr\n5hMREZGeGMICsJlNcLJiPhEREemEISwAk0lAGt0IIiIiWrYYwoIQRjeAiIiIli2GMCIiIiIDMIQR\nERERGYAhjIiIiMgADGFEREREBmAIC4KrI4mIiEgvDGFEREREBmAIIyIiIjIAQ1gQrBNGREREemEI\nIyIiIjIAQxgRERGRARjCiIiIiAzAEEZERERkAIawIFgnjIiIiPTCEBaClIxiREREFH0MYUHYzAJu\nL0MYERERRR9DWBB2ixkur8/oZhAREdEyxBAWhN1qgtPtNboZREREtAwxhAVht5jg9LAnjIiIiKKP\nISwIu8XMEEZERES6YAgLQvWEcTiSiIiIoo8hLAibxQSnmz1hREREFH0MYUFwOJKIiIj0whAWhN3K\n4UgiIiLSB0NYEHaLCS72hBEREZEOGMKC4HAkERER6UXXECaE2CWEaBBCNAsh7tW4vkQI8YIQ4pAQ\n4qgQ4iY92xMpro4kIiIivegWwoQQZgAPAbgRQDWAu4QQ1XNu9lUAj0spNwO4E8B/6tWehVAV89kT\nRkRERNGnZ0/YVgDNUsrTUkoXgMcA3DbnNhJAytT/UwF06NieiHE4koiIiPSiZwgrBNDqd7lt6pi/\nrwN4vxCiDcBuAH+ndSIhxN1CiFohRG1vb68ebdVk43AkERER6cToifl3Afi5lLIIwE0AfiWEmNcm\nKeXDUsoaKWVNdnb2BWucncVaiYiISCd6hrB2AMV+l4umjvn7KIDHAUBK+TqAOABZOrYpIhaTgNsn\njW4GERERLUN6hrADACqEEGVCCBvUxPsn59zmHIBrAUAIsQYqhF248cYQhBAQRjeCiIiIliXdQpiU\n0gPgHgB7ANRDrYI8IYR4QAhx69TNvgDg40KIIwAeBfBhKSW7noiIiGjZs+h5cinlbqgJ9/7H7vf7\nfx2Ay/RsAxEREVEsMnpiPhEREdHbEkMYERERkQEYwkLgBDUiIiLSA0MYERERkQEYwoiIiIgMwBAW\nAuuEERERkR4YwoiIiIgMwBBGREREZACGMCIiIiIDMIQRERERGYAhLATWCSMiIiI9MIQRERERGYAh\nLASTAHw+9ocRERFRdDGEhWC3mOHy+oxuBhERES0zDGEh2C0mON0MYURERBRdDGEh2K0mOD1eo5tB\nREREywxDWAh2ixlOD3vCiIiIKLoYwkKwW9gTRkRERNHHEBaCzWLCJOeEERERUZQxhIWgesIYwoiI\niCi6GMJCUHPCOBxJRERE0cUQFoLdaoKLPWFEREQUZQxhIXA4koiIiPTAEBYCS1QQERGRHhjCQlAV\n8zknjIiIiKKLISwEVTGfPWFEREQUXQxhIdjNHI4kIiKi6GMIC4F7RxIREZEeGMJCsJlZooKIiIii\njyEsBJNJQEqjW0FERETLDUMYERERkQEYwoiIiIgMwBBGREREZACGMCIiIiIDMISFgfPyiYiIKNoY\nwoiIiIgMoGsIE0LsEkI0CCGahRD3alz/70KIw1NfjUKIIT3bQ0RERBQrLHqdWAhhBvAQgOsBtAE4\nIIR4UkpZN30bKeU/+N3+7wBs1qs9iyGMbgAREREtO3r2hG0F0CylPC2ldAF4DMBtQW5/F4BHdWwP\nERERUczQM4QVAmj1u9w2dWweIcQKAGUAng9w/d1CiFohRG1vb2/UG0pERER0ocXKxPw7AfxOSqm5\nU7aU8mEpZY2UsiY7O/sCN42IiIgo+vQMYe0Aiv0uF00d03InOBRJREREbyN6hrADACqEEGVCCBtU\n0Hpy7o2EEKsBpAN4Xce2LArrhBEREVG06RbCpJQeAPcA2AOgHsDjUsoTQogHhBC3+t30TgCPSSmZ\ndYiIiOhtQ7cSFQAgpdwNYPecY/fPufx1PdsQLVJKCMFiFURERBQdsTIxP6bZzAIeHzvqiIiIKHoY\nwsJgt5jh9PiMbgYREREtIwxhYbBbTXC6NatnEBERES0IQ1gY7BYTe8KIiIgoqhjCwsDhSCIiIoo2\nhrAw2CwmOD0cjiQiIqLoYQgLg91igtPNnjAiIiKKHoawMHA4koiIiKKNISwMdqsJLoYwIiIiiiKG\nsDDYOSeMiIiIoowhLAwcjiQiIqJoYwgLA3vCiIiIKNoYwsKgKuazJ4yIiIiihyEsDDYzK+YTERFR\ndDGEhcFuNXM4koiIiKKKISwMLNZKRERE0cYQFgaLScDjk0Y3g4iIiJYRhrAwCCGMbgIREREtMwxh\nRERERAZgCCMiIiIyAEMYERERkQEYwsLEaflEREQUTQxhRERERAZgCCMiIiIyAENYmFikgoiIiKKJ\nIYyIiIjIAGGFMCHE54QQKUL5qRDiLSHETr0bF0s4MZ+IiIiiKdyesP8jpRwBsBNAOoAPAPi2bq0i\nIiIiWubCDWHTU6JuAvArKeUJvM2mSZmFgJf7RxIREVGUhBvCDgoh9kKFsD1CiGQAPv2aFXvSE60Y\ndLiMbgYREREtE5Ywb/dRAJsAnJZSOoQQGQA+ol+zYk9Gog0D4y5kJdmNbgoREREtA+H2hF0CoEFK\nOSSEeD+ArwIY1q9ZsScj0Yb+MfaEERERUXSEG8J+DMAhhNgI4AsATgH4pW6tikGZiXYMjDOEERER\nUXSEG8I8UkoJ4DYAP5JSPgQgWb9mxZ6MRBv6x51GN4OIiIiWiXDnhI0KIe6DKk1xhRDCBMCqX7Ni\nT3qCFYPjbqObQURERMtEuD1h7wXghKoX1gWgCMD/1a1VMchiNsErWaKCiIiIoiOsEDYVvH4NIFUI\ncTOASSllyDlhQohdQogGIUSzEOLeALe5QwhRJ4Q4IYR4JKLWExERES1R4W5bdAeANwG8B8AdAPYL\nId4d4j5mAA8BuBFANYC7hBDVc25TAeA+AJdJKdcC+PuIvwM9jPcb3QIiIiJa5sIdjvwKgIullB+S\nUn4QwFYAXwtxn60AmqWUp6WULgCPQU3s9/dxAA9JKQcBQErZE37TdbT/J5qH31ZbBBAREZGuwg1h\npjkBqT+M+xYCaPW73DZ1zF8lgEohxGtCiDeEELu0TiSEuFsIUSuEqO3t7Q2zyURERESxK9zVkc8I\nIfYAeHTq8nsB7I7S41cA2AE12f9lIcR6KeWQ/42klA8DeBgAampq9J8dH58OOAaAhIxZh4UAfD4J\nk4l9YkRERLQ44U7M/xJUCNow9fWwlPKfQtytHUCx3+WiqWP+2gA8KaV0SynPAGiECmXGyqoE+prm\nHU6Nt2J4gmUqiIiIaPHCHY6ElPL3UsrPT309EcZdDgCoEEKUCSFsAO4E8OSc2/wRqhcMQogsqOHJ\n0+G2STdZFUBf47zDqmArq+YTERHR4gUdjhRCjALQGv4TAKSUMiXQfaWUHiHEPQD2ADAD+JmU8oQQ\n4gEAtVLKJ6eu2ymEqAPgBfAlKaXxSxNTi4HhtnmHuXURERERRUvQECalXNTWRFLK3Zgzd0xKeb/f\n/yWAz099xQ6TCZC+eYczEm1o6R83oEFERES03IQ9HElAZhKHI4mIiCg6GMICsdgA9+SsQ+kJNgwy\nhBEREVEUMIQFklEODMxeI2CzmOD2cf9IIiIiWjyGsECyKjVXSLJCGBEREUUDQ1ggmeVAf7PRrSAi\nIqJliiEsEGs84HEa3QoiIiJaphjCiIiIiAzAEBaKb369MFXejIiIiGjhGMKCSS0ERjtmHUqOs2Bk\n0mNQg4iIiGi5YAgLJqsK6G2YdSgzycati4iIiGjRGMKCyaoE+ppmHcpItKN/jBP2iYiIaHEYwoJJ\nzAQcs/cTz0zk1kVERES0eAxhoYjZ5VkzEjkcSURERIvHEBbKnJWQDGFEREQUDQxhocSlABND5y9a\nzXB55petICIiIooEQ1goWZXcvoiIiIiijiEslAAbeRMREREtBkNYKGklwGCL0a0gIiKiZYYhLBST\nGZCcA0ZERETRxRAWjjllKgDuH0lERESLwxAWDpMV8LpnLibazRh3eQ1sEBERES11DGHhyCgDBk6f\nv8iti4iIiGiRGMLCkVUxa4Ukty4iIiKixWIIC0dmxayNvDOTbBgYYwgjIiKihWMIC4ctAfBMzlzk\n1kVERES0WAxh4fKdn4ifmWjncCQREREtCkNYuFIKZoq2xtvMmHRzdSQREREtHENYuKpvA+qfNLoV\nREREtEwwhIUrMQtw9BvdCiIiIlomGMIikVUF9Jw0uhVERES0DDCERWL1O4CTTxvdCiIiIloGGMIi\nEZcCOEcBKcGdI4mIiGgxGMIiVbgFaH8L8VYzHC6P0a0hIiKiJYohLFIVO4GmvchMsqGfVfOJiIho\ngRjCImWNA3xuZMabWTWfiIiIFowhbCFWXIqSsUMMYURERLRguoYwIcQuIUSDEKJZCHGvxvUfFkL0\nCiEOT319TM/2RE3ZDuT0vsGti4iIiGjBLHqdWAhhBvAQgOsBtAE4IIR4UkpZN+emv5FS3qNXO3Rh\ntiDeZsXQ6JjRLSEiIqIlSs+esK0AmqWUp6WULgCPAbhNx8e7oKxV1yO7+9VZx5weLz776CH0jE4a\n1CoiIiJaKvQMYYUAWv0ut00dm+tvhBBHhRC/E0IUa51ICHG3EKJWCFHb29urR1sjJoq3IX/02Mzl\nSbcXD+4+iV3r8nC8fdjAlhEREdFSYPTE/KcAlEopNwB4FsAvtG4kpXxYSlkjpazJzs6+oA0MyGSC\n2xQHTI5gwuXFg7vr8cmrynHN6hzUd44a3ToiIiKKcXqGsHYA/j1bRVPHZkgp+6WUzqmL/wNgi47t\nibqGrJ1wHX4cD/6lHp+5ehXyUuMQZzXD6fEZ3TQiIiKKcXqGsAMAKoQQZUIIG4A7ATzpfwMhRL7f\nxVsB1OvYnqjrsRbizYO1+LurVyEnJe78FZKbGhEREVFwuq2OlFJ6hBD3ANgDwAzgZ1LKE0KIBwDU\nSimfBPBZIcStADwABgB8WK/26KFmRTo25FyHlJHjQErNzPH0RBv6x5zITLIb2DoiIiKKZUIusV6b\nmpoaWVtba3QzzvO6gRcfBK69f+bQgbMDmHB5cWVljMxfIyIiIkMIIQ5KKWu0rjN6Yv7SZ7YC1nhg\n8vyKyDX5KajrHDGwUURERBTrGMKiYf17gGO/nbmYZLfA4fQY2CAiIiKKdQxh0ZBeCgy2cEI+ERER\nhY0hLFqKLgbaDsxcTI6zYnjCbWCDiIiIKJYxhEVL1Y1Aw19mLq4tSMGJDlbOJyIiIm0MYdFitgK2\nRGBiEACwtjAVdR2cnE9ERETaGMKiacMdwLHfAQBS460Y4XAkERERBcAQFk1pJcBwKyfoExERUUgM\nYdFWuAXoeAsAEG+zYJylKoiIiEgDQ1i0VdwANO4FAFQXsGgrERERaWMIizZrHODzAF4P1hak4Hg7\nV0gSERHRfAxheii7EjjzErKS7BgYdxndGiIiIopBDGF6KL0caHnN6FYQERFRDGMI04PJDJjtgMsB\nu8WESbfX6BYRERFRjGEI00vVLqDxL1iTn4KTXaNGt4aIiIhiDEOYXvI2AF3HsK4wFcc4OZ+IiIjm\nYAjTixBAfDpyzOPoGZk0ujVEREQUYxjC9FR9G0T9n4xuBREREcUgi9ENWNbSS4Ghc7CYroDL44PD\n5UFd5wjqO0fRPTKJWzYUYH1RqtGtJCIiIgOwJ0xvqUW4Jm8C33+2Eb872AYpgds3F+Kjl5fhaPtQ\nVB/qV6+fjer5iIiISD/sCdNb9e1Yf/h/sf7Gz8067Pb60D8W3UKuv95/DndtLYHFzGxNREQU6/hu\nrbfETMDRD0g567DVbILHJwPcKXJen8Sk24tzA46onZOIiIj0w56wC2HF5cDz3wBM1vPHpBfFQ2UA\nKqPyEN0jk7i8IguN3WNYmZ0UlXMSERGRfhjCLoTKneprjpX/9UnAdxdgWnyH5LkBB66uymFhWCIi\noiWCw5EGasy6Fqj7Y1TOdW7AgcrcZDi5RRIREdGSwBBmoM6UjZBttYDXs+hzdQxNID81LgqtIiIi\noguBIcxAqfFWjFXeDhz77aLP5fNJWMwmmEwC3ihO+CciIiJ9MIQZKDvZju6k1UDPCcATnXIVJRkJ\nXCFJRES0BDCEGSgn2Y6eESew8X3AkUeics7K3GQ0dXNyPhERUaxjCDNQdrIdvWNOILcaGDgDuBe2\n0feY04NEu1roWp6dhObesWg2k4iIiHTAEGag7GQ7eked6sJFHwQO/WpB5znX70BJRgIAIN5mxqTb\nF60mEhERkU4YwgyUbLdgZHJqZWRmOTDaCbjGIz7PuQEHSlLNgGMgyi0kIiIivbBYq4GEELMPbPkw\nsPdrQNVNQPnVgMk8/05eD3D2FeDcGwDUKsjsliGUl2QCx3qA7Z+GSahtjMwmMf/+REREFBMYwmJJ\nWglw03eBlleB578JWBOAte8EMlYC514HTr8EQAKllwNXfgkwq1/fS882Yss1lWqF5d6voDjn79A6\n4EBpVqKx3w8REREFxBAWa0wmoOxK9TU5oirqH3kUKLkUuOofAbN1/n2mNwe32IDNH8C2Y39CfcId\nDGFEREQxTNc5YUKIXUKIBiFEsxDi3iC3+xshhBRC1OjZniUnLkVN2L/2fqDiOu0ANlf+BuTYXeg5\nW6d/+4iIiGjBdAthQggzgIcA3AigGsBdQohqjdslA/gcgP16tSWW2cwCLs/CVzN6fXLe3DLbZX+H\n6tZHAB9XSRIREcUqPXvCtgJollKellK6ADwG4DaN230DwL8BWFiRrCUuK8mOvjHngu/fNTKJgrQ5\ne0ZabDiRczPw1s8X1zgiIiLSjZ4hrBBAq9/ltqljM4QQFwEollL+Wcd2xLRZtcIWoKV/HMVTNcL8\n9Satgc8xqIrAEhERUcwxbGK+EMIE4PsAPhzGbe8GcDcAlJSU6NuwCywnOQ5dIwvvBGwdcOCyVVnz\njhdnJKCt+P+g5LWvA9mrAc8k4HFOTeKXQEIWULIdyF2rXQqDiIiIdKVnCGsHUOx3uWjq2LRkAOsA\nvDg1pykPwJNCiFullLX+J5JSPgzgYQCoqamROrb5gstOtuNY+/CC7985PIn81Ph5xytyktDY50TJ\nDQ+qAGaNB8x2tfoSAEa7VK2xk08DPi9gSwA23Amknu+sHHK4MO7yojBt/vmJiIhocfQMYQcAVAgh\nyqDC150A3jd9pZRyGMBMF44Q4kUAX5wbwJa7zCQb+hcxJ8wnoVmUdVVOEl4/3Y/rqnNVwJorOU/V\nIFv7TnXZOQrs+yGQVQmsfzcA4MWGXtR3juC+m9YsuH1ERESkTbc5YVJKD4B7AOwBUA/gcSnlCSHE\nA0KIW/V63KXGajbB7Yt+516i3QKH0xP+HezJwNVfBhIygD1fARwDON07BrNJwOPlKksiIqJo03VO\nmJRyN4Ddc47dH+C2O/Rsy1I2MO5CRqJN+0oZ5QBXfg1QsBl45XsoGViFwotvwavNfdixMhlw9Ksv\nayKQtSq6j3sBSDm/nAcREZFRuIF3jBtzevCp/z2oed3IpBtJcYFztBACvoX0ssWnw3nNvyARDmw/\n+2NYX/428MaPgVPPA8NtQOt+4K9fB176DtB2cEnUI3N7ffjW7nqjm0FERDSD2xbFuIauUaTEW9HS\nP44VmbO3IWodcKAkI/DWRIXp8WgfmtAsYRFKfecoLBv+BqI6FwfRhE1by5Bon/N0cY2rYPbCnwGT\nFVhzC5C3LvwH8XqAtjeBs68BSTlAUY1ayanDas2u4Umc7BqN+nmJiIgWiiEsRgQaKjvZNYIv7qzC\n00c78ZmrZw8BhgphlbnJaOoZXVAIO9I6hBvX5wEAblibh711Xbh9c9HsG9kSVfBacwvgngDqngSO\n/kZtOL7uXUBc6uzbez3AaIfaiHzwrApbRVuBS+8BxnuBtlp1DulVoW7Lh4Hk3IjbrqWtbxCrx94E\nsG1hJxg4A6SXAhzOJCKiKGEIiwEpcRaMTHqQGj9/b8ju4UlU5ibhiUPt8PkkTH4rIVv6tWuETVuV\nk4Rfv9GPa1ZHHmT6xpzISVaV+KvykvHEofbgd7DGAxvfC+C9QP8pYP/DgGcCMPk9xYRZrcosuwK4\n6AOz759Wor6mOceAl74NVO4CSi+PuP1zpb75H7jY6oa7+SVYV10V2Z1d48DerwLV7wQ2vGfRbSEi\nIgIYwmJCTkocekcnNUMYoOZ2bVuZgf1nBnBJeebM8XGnB8lxgTf1TrJbMO7yLqhNc+f7Zyfb0TM6\nORPMgsosB6760oIed4Y9Cbj+G8ChXwGdR4Btnzpf4yxSJ55As201XNt2Yuzkw0hPL1JtDNeB/wFu\n/Dfg0P8CQ61AWnHo+xAREYXAifkxIDvJjh6NrYukXxK6YlUWXmnqnX0DnYbGhhyueYHw5g35+PPR\nTl0eLyAhgIs+CKy4DNhzH+AYiPwc/aeAviY0p1+O8uxEHFn5CWD/T4DJMAvkjnYBPg+QWgRc9jlg\n3w+WxEIEIiKKfQxhMSDQ/pFtgxMoTFfV6i1mE+wWM8Yjqf0FIMFmjrgY7JG2YWwsTpt1LDclDt0j\nCy8quygFm4Ad9wGvfA944ydA51FV5T8U96Ra1Xn5PwBSojQzEWcGXepcz/+rmqMWypsPAxd/XP3f\nGg9sfj9w4L8X9/0QERGBISwm5KRoh7CTXaNYnZcyc/nG9Xn4y/EuAIDH64NGofx53rOlCI++eS6i\n9hxvH8a6wpR5x1flJKGp26AVhvFpwA3/Cqz7G2DgNPDit4HnvgHs+xHQfUK7Xtor31W9V2bVq5eW\nYMWgw60K0l78MXV9MF3HgdRiIM7vZ5G/UQXA7hNR/OaIiOjtiHPCYkCyXU3Mn6uxexSX+028r8xN\nxu/fagOg9ows0Ngzcq7MJDs8PolhhxupCYHnj/mbdHuRYJv/1LhhbS5+se8sKnKTwzqPLpKyZ2+3\n5BhQZTKO/RawxAOrrgMKLwLq/ggUbgHSiuHy+GA1m2avPs2uBIouBg78FKj5P9pDu4d/realzbXt\nE8Az9wI7vwlY7OqYexJo/AvQcRgw29T50svUBunZVedvR0RENIUhLAYEquLudHsRb5tdM6soPQGt\nAw5VniIzvNIT79tagkcPnLBKnm0AACAASURBVMMnrwo9GV0GqcCfHGfFuMs7b5WmoRIy1F6X698N\nuBzAqeeA554GEjJV6QsAncMTKNDahHzVtSrA7fkKsP1TsyfcN/0VWLkDMGv8iZjMwCX3AK/+P7XS\ns/k5tQq0apdaQSmEmjc2eEb1mDXuAZzDKuxlrNTlx0BEREsPQ1gM04pDN6/Px2MHWpGeYMWVldlh\nnScnJQ4Olxejk+6gqykBNQ+tOD1wuLu4NB0vNPTgmtU5sbcFkC3hfN0yP60DEyiamltnFmovTIt5\naiS+/BqgeDvwxn+qumZbPqKON/8V2PVg4MdKXwEUX6w2Pt9x78yQ5wyTSa3AnF6F6Z4Ean+qyl1s\n+8T8GmoUHimjtyBl4Azw4oMqgBdsjs45aWlxDKgPckQGYQiLURMuL+Ks8yvHpyfaMDThwqTbi9yU\nMMpFTLnz4mL85kArPnZF8J6YQ61D8ybl+7uyIht/PNyBf3+2EQBgs5iwtiAVW0rTkRIi4BmlbdAx\nE1gL0+PRMTQ5uxfRlgBc+UXVa/XMvaoI7ab3hX6zL78m/EZY44BLPgOM9QCv/UD1um16v3ZP2zQp\nVcjzeWLnjWK0WxXkTSsBMlepnj3b1M/S51WrUbuOAn1NgGdS9TaWXbn4x3VPAM89oB5jzc2LP+fg\nWWD/fwG3/lAF8NEuoOrGiE7x89fOYOfaPO1eVoptfc3AoV+qfXCdI8D2TwOphUa3Knwe5/lpD7Sk\nMYTFiLl/So3do6gMMPdqa2kGfr7vLMwRDAkWpMVjyOGGw+XRnO81rblnDO9Ynx/weovZhHdvOV85\nf9LtRV3nCP7jr0342s3VYbfnQuoamZwJrKWZCTjbP649lJu7FrjxO0D7QSB/Q8jznukbR0lGQkS/\nByTlANd+Deg6Brz8f3G+v1NAs+/TnqyCx8QgUH2rmue2UD6vetyzr6o3nrmsCcDWj6sQqqVxL3Du\ndRUmR7uA/magaY/q5QPUG0JGuVq8UP1ONWx78s/AM/cBa9+leg4XYqxX9Vhd+SVV7Lf+KTWEfNGH\n1Ny+aa5xNfTbeUS9Sa1/D1Ck8fMaPKtWzV7/DcBiU6tnDz8K1P5MDRlrcTnUkLPFBgA4cHYAdZ0j\nWJGZGJ0Q5hoHOg4Bgy3AikveHsPWk8Nqkc3AGbVdWe4CXj/GeoG+RhVILDbAEqfmX1oT1ZSEubUF\nB84Ab/0SSCkAdnxZfThyjavngy0JuPij83u1RzrUtAXnmOrB9v/Kqpx5TujONQ40PqNWh1vi1Icz\nQH2fJduBvA3BP9QtxMQg0Nugvkba1QdD17j6ILTi0sjO1XkUOPa4+l2lFgMV16vSP29zItgcoFhU\nU1Mja2trjW5G1P3wuSZ84qpy2CzqReM3B87h0vIszS2H3F4fvvl0Hf7ltgj2aQRwrt+B509248OX\nlQW8zff2NuALO6siazyAX+w7i2vX5KAoyFCmUb7/bCM+f716s+4ddeKZ4534wCWliz7v539zGJ+5\nZhXKs5MWfa6QvB612KD9LTUPreIGFXpGO1UY6m9WvVSAKnQblwbEp+GphjHszB6EfbIPECb1Ql16\nuXbP2nCb6h3KWw+se/f5NzCPE3j139Ub5fSCiEj4fMCJP6iQseG9YQXcGX1NqkzItferQDrN4wLe\n+oXa7ipnjQpe1gSgYqcKgVICRx5Rb7qX3gPEp6v7Dbaonq/pAObv9EtAyz7gqn9UAXKkA2j4CzDc\nqt7UAcDnwYTLgwNnB3BxWSYODdhw6fZLgawqIDErvJ4Jj0v9vrqPqwABqPInBZuBtBVAy2sqKAqT\nOlZ6ufre/EkJuB3qDdE1DrhGVW9hcr7aXktr/9WxHuDMy0BPvWpn8Xb1XIpk0YiUwFCLamc436vP\np27fe1K9kbvGz9/PngJklKn2dh0HeupUyF77rsDblY31qA8RPXWqLYlZ6vfv9aieV69TPV+dY8DE\nACD9avp5nCp8XfRB9fOeq+ekCuJrblEB59wbgNel7lN+jXoOTQ6p8Dg5rALK2dfUh5KMAK+pPp+a\nhtDfDJRfq4JHuL1X7gnVs9xTD/ScUM+ByhvU37D/OcZ6gdb96gOW/wc5YVI/H2uiOi7l+X8Ts4DC\nGrXQSetnXPcn9Xow/fPNXg2kFKrHlRI48qgqXH3ZZ7V/lv76TwEHf64WKG24UwXFwRY15WOkQ4Xe\njHL1ATUpV/0bnw543ep503VM/T0AaheW0ivU33mkvYBjPYCjH5gcUSMMzmH1M978/sjOswBCiINS\nyhrN6xjCYsOjb57Djqps5E+tePz+3gb8w/WVUZ939W/PnMTnrq3QHOp0e3146IVm/P11lRr3DG7I\n4cIjb57Dp3esCn3jC8w/hEkp8e9/bZq5vFBSSnz8lwdx++ZCvGND4J7DqJNSvQmdel69eCXnq2HB\nzHL1f0C9wEwOARND+Nc/7Mf7broaZWUR/F7O7QeO/x7YcIcKPm/+t3qx9d9WaiG8HjWU2VsPbLxL\n9TwG07JPvVDv+HLgT/iOAfUmn79J+0V5vB94/YdATjVQvC1wAJvWc1IV841PVz/Pql2zvm8pJf7l\nqTp8fmclkm1mPLx7Hz5R7QF6G1UghFTbc52/hwo5Pq96Y5dS/d4yy4HcdUBmReCdIHxeFVxb9qlw\nMZc1UfVa2hJVL441Tr2pDZ49Hz7sKYBrTD12Yo4axs1Zo87dul+FMq9L/XxWXRt42NvjVCuQe+pV\n24fbAAhVw2/ljqkeWx/Q36R6kgdOq/sJs/r5Za9WvZb2ECurRzqAE08AY92qt0edBDPhIiFTFW/O\nqV74DhrBSAmcfFr93Eq2hw6oHpeqX5i3bt5cVIx0qMU7m+5Sz89Tz6vnc/m16mft/3wd7QbO7VM/\n32kWuwon2VXq5xfpe4HPC4z3Ae5xAGLq/lP/jvUC7bVTz1mon6s1Xn1oScxWve6h/t6H24B9P5zq\ncZ6TL6YXJh1+RIWqiz6knp9aPE5g6Jz6nY91q7A0MaR6nrOr1IfCtBXq9y0lcOYloOlZNSpQfdv8\nDxxSqvN0HlFTIzxTfztJuSpU2pMBe+rUv8kXZBiaIWwJeK6+G9nJdmwoUvOxvr+3AZ9fQI9UKKd6\nx/D6qX68f/uKedcdbx9G2+AEdq3LW9C5/+2Zk/jC9ZXnJ72H4bXmPvz+YBu+ftta3eaU+YcwrcsL\nca7fgZebejEw7sJnr62I+P4+n8SoU3u/0GiRUuKu/34Dn96xKuxFHDN8PjV0MN4HbPtkdIc5vG71\nSbqvEdj4vtnDUGM9aiP3zsMqCG37ZHTmvbS8roZOd9y3qHIhv9h3FptL0sL/O5VShRxhmj/MdSFM\nDqselGCPLaV68z/13NSbn1m98a24TP3sD/1a9SCsf/fs4OzzAZ2HgNMvquFaYQKyKtSbY8bKt9d8\npfqnVI/NFV9Qz68Tf1Q9nZd//vycSUD9rJufU2Ess1wN6wMqIKy4VL9wGcp4n5qiEOkwuJRqa7mB\nM+o5Np0nhFBDjdW36bcIqfVN1WM3PWVj+nEBFSTzN6lwHKqn7gIIFsI4JyxGZCfb0TNVkV5Kqbky\nMhrKs5Pw29o2jEy654WeQ61DuH5N5Jt9T7u6KgcvNvTiuurQ52juGcWjb7bi4tIMfOGGKjwexqKB\nhZh0e2G3RP9F7Y3T/biyIhu/O9i6oPs39Yzh0TfP4eu3hugNWoRzAw5sK8tE1/Bk5Hc2mYCNd847\n7PVJ7DnRhZuCzBsMyWxVQ0IelxoyPPzr88Nt08Mk029m0bLiEvW1CG+dG4RJYCaAhUUIY2vEhfMG\nKIQKwtNh2OtRPQiHH1FDfJvep4bk5jKZVOBazDzF5WLNLSq47vmyei6XXgFc89X5txMCqLhO9YSN\ntJ8f4jNaYpb6itT01nIuh+q5vJABsnir+nJPBu5lWwIYwmJEdrIdJzrUZOmukUnkper3pPrkVSvx\n/b2NuKOmGNUF56vB9yzycS8uTce3nzkZNIT1jTnxi31nkZ1sxz/tWj0zB65/3AWnxwu7RWMuyyJ0\nDE2gcM7EaQEVJgJNqG/XuM9c5wYceE/NwieVnuodQ+uAA1JK3Up9HGkbxvXVuXjhZE/Uztk+OIHd\nxzoXF8KmWWzAlg9Ht+yEToYdbjx1pAP3z1l8YjGb4PaqYsDLhtmiCh4XXmR0S5aW9FLghm+peUbx\nIYL6dE/RcmEzcC7wEg5gALctihlZSXb0TW1ddLJz9nZF0ZaWYMP9N1fjteY+/GGqAn80CCGQlxKH\nzuEJzev7x5z4wXNN+NjlK/HBS0pnAhgA3LqxAH863BG1tkxrGzxfI2xaQVocuka0e4eklPjozw9g\n0h14b8rpIXwhBBLtFoxOuiNu15m+cexal4e6To1VilHS3DOGNfkpcPui1696qncMoxq7OyxKjAcw\nAPjVG2fxuWsr5gXmovR4tA1qP98vtG/trl9YrydFj8UeOoAR+WEIixFWs2nmzfJk1yiq8vTdGshk\nEvj4lSuRnWzHd545id5RJ5Lsi+8YvW1ToWaY8vkkfvh8M750Q5Xm9klr8lPQ2DUKXxQDAwC0Djrm\nrdhckZmIlr5xzduf6h3H9pWZeLEhcO9R68AESqZWrVbmJaNxAftpOj0+7FqXh5caeyO+b7ikDNzb\nt1CnesdQlhWghMUy5vT4kJYwfzL/isxEnO3Xfi5dSFJK9I068Wxdl9FNIaIIMITFoAmXJyqBKBxX\nVGTj/dtX4P4/HQ9apDVcGYk2DE+44Z0Tpn722hm8p6YoaMX+a9bk4IUg4WchekacyEmePSenNDMR\nZ/sdmrd/takXn95RjqNtwwHP+cbpfmxfmQkAWJOXgpNdC9vUPDnOirFo9ypN8fr0GeYcmXCjOCMB\nQw5X1M8dq4ItXirLSsTZAIH+QmruGcMVlVnoYE8Y0ZLCEBZDpt8yL/R61YK0eDz0vouwrSw6Vdmv\nqMjCy03ne3jeON2P5DgL1hYEnyR8ycpM7DvVH5U2TJPAvH0uc5Lt6A4wHNk75kROShzsFnPAIclz\nAw4UZ6ghztwUO7ojfOPzf1MvSItH26B2IFyMU71jKM/Wp8eqNDMBLQFC7HLUPeIMuDtFeoIVg+PG\nB9KXGntxZUU20hOsGIiB9hBReBjCYoiEWs1nM2CSr8kkotZzcsnKTLxxWoWp3lEnnj/ZgztqikPc\nS82x2lyShrfODUalHYGYTEIz6Do9XlimVvdcVZWNFxvmDxX6zwfz/zcSvWNOZE/1zl23JhfP1Ue3\n9w8AjrYNY+PUKr4Emxnjzuj1uK2Y2nXg7eJExzDWFmjP0YyV/VMHHS5kJtlx3Zpc/LW+2+jmEFGY\nGMJiTHPPGCpyL0AFdh0JIZCdZEfn8AR+9HwTPqsxoTmQXWvzsOe4MfNaDrYMoqZUVVbfWJSKo21D\n827jPx/MXyT19k71jKN8al5VXmpcwF656fO+1twX9rmntfSPY8XU1kz5qXHojMIw1eC4C6kJNhSl\nJ8TMZPQL4WSXvgtlFmtk0o3EqekLK7OTcCYGhkeJKDwMYTEkJc6CA2cHYvoFP1zv3FyIT//6Ldy5\ntSSi+W0WswlFGQk43Tu26DZMur2IswZ+is8NTgfODOLiUjUkK4TQHJL0nw82rSAtPqK5OKf7xrDS\nb6ujlHgrhh3aKyyfPtqJX75+Nuxz+5sOvvmp8VFZNXe6Tw1xxlnNcAZZPbrcTLq9iLcFLp0yXabC\nKK819eGKVeeL8SYtcMUuEV14DGExJDvZjtqWQc2elqUmK8mOh953EdbkRx4o331REX4fhdIZbYOB\n633lptjRMzp7KxiX1ztrOyetIUn/+WDTVuen4GQEpSbUHKPziwWurtJekDDh8uJo2xBW56XMW+gQ\njMvjm7UqMj81Dh0ByoZE4lTP+Pl9MmNkGC4WRLNMxUJ2MDnRMTJruFQ9n/RbdUtE0cMQFkOyk+0Y\ncrjmTSRfqgpCFDwNJN5mRl5qPH5/sC2i8DFX66BDcwN0YGqFpN+wTd+YE+lzShDMHZKcOx9sWmVu\nEhoiLFPhf47K3CTNMhe/eP0sPnRpKVZkJqAlgjlYjd2jWO1X4iRnAYsHtLT5F7Fd4HZn33i6Liq9\nnBfKkMMVcmupaJWp2H2sE/c8eiiiIDZd0sX/NWNNfnJEHwqILpR9p/qCTr94O2IIiyE5yXEoyXj7\n1WDS8oHtK1Cek4RvPF2HV5oW9qleq1DrtBVzVvi91tyHKypm7684d0gy0HywBJsFE66FD88JIWA1\nm+D0nD9H+9AEpASK0hNQlZeMhgjKYBxtG561tY7dYo5OwVYpZ97s42xmOFyRTfY/2DKA9YWp+OXr\nLegd1diQOgbVdYzM2lVCSzTKVDz65jm4vT7cUVOMFyOoHVfXOTJv0UCgoXSarXfUifv+cGxRH/TC\nMeHy4vUor/peql5u7MOjb54zuhkxhSEshhSkxeGWDVHYDmaZ2FSchn++pRoTLi8eeKoOJ7si+3Tf\nN+pEdpL2vn35qfGzhugau0dRqbEg4srKrJmCqlrzwaaF23cZaC/LS8tnl+f4xb6z+PClpQCAVTlJ\naO4Jv/eoc3gC+TpuewUAKzIScW4g/DIVUko8fbQTt24swD/uqsJ/PNcY1RWbejnRMYK1+cFLqyym\nTIWUEj9+8RTyUuNw26ZCXFmRhVca+8LuDXulqQ+XVczf8++Kyiy8rGMh4KXuSOsQ/uulU9i5Njdo\nYeZoeOpIB5453qnrYywVVrOAxSRYRsUPQ1gMSbBZcOmqBWyiuowJIbBzbR7uu2k1Dp0bwvf2NuB7\nexvwyP5zONo2FHx7IQQuIWA2CUx/AA40zAioIHikVQ1Jas0Hm2azmMLqeWjpd8ysWvS3ZUU63mpR\npTn2n+7H+sLUmcngdosZrggnfke7dILT4521P+LcnsRQXmzoxY6qHJhMAgk2C/7+ukp855mThk5o\nD8fIpFtzhwd/C/1Ze30S393bgK1lGbi6KmfmXHMDeTBjTjdSNAogbypKw+HW+at7CXjiUBtqWwbx\nlXeswY7KbLx5ZkDXx2vuHQs5pP12MDDuQlqCDXduLYlqb9hC5lHGEm7gTUuC1WzCXVtLAKg/us7h\nSZzoGMHzJ3tQkpGAd1208M1wG7pHUZmrvU2UEGJWwAr0hjvdW7WuMHivyenesfOT2/1YzCb4pITH\n68Nfjnfhn2+ZvVH0YnvaFutcvwMr/LYrWpGZgH2nwiud4fNJvNLUh6/dvGbmWFaSHR+5rAzf29uI\nf9pVNfNzbRt04OmjnegansT2lZnYtS4vut+IAdqHJvA/r5ye2S3CJIDUeCta+h34220lqJjz3Lt2\nTQ6++ed6XBbiA9nAuGvePMZpJpPqcVh2m4svgtvrw4+eb8amkjTcvvn860Xu1H63+akLm8MazJHW\nIawvTEVT9yik1GcXi6XirZZBXFSShqwkO1weH0YmtT9ARGJw3IXv7GnAg+9aH6VWXngMYbTkCCFQ\nkBaPgrR4XF+di2/trl/wm42UEq829eH2zYUBb3NlZTb+942WoKtWV+el4GDLYOgQ1jeOq1fnaF63\noSgNX/vTcfztthXzXqztVjUHK8EW/E/2RIA5TPFh3j+QuRX4k+OsGHOGN+foySMduGVj/rzvqTQr\nEbvW5eGhF5qRnxqPpp4xFKbF4Y6aYmQk2vDg7npcUZE1UwPrQptwzV4tG8x0mQqt5+DeE1341I5y\n5CSrIWKvT2J4wg2rWWhu4yWEQM2KdBw4OzBTMkXLy429uKoyO+D128sz8fqpflwZ5DZvJ9/d24D3\n1hTPKg8DqHI6vznQik/tKI/6Y+450YW/v64SA+Mu9I+7kBVgesS0/af7sS3AlIel7njHMD69YxUA\n4M6txXj8QCs+dsXKRZ3zj4fbY743PRR+RKIl77ZNBfM2DR93epAQpLYTAGQl2dA/7pqpNh7I5uI0\n/O5gW8D5YABQkpGAljDmSDndgd/Yr6zIRl5KvGaQq8xNRlN36Hlhx9qGsL5w/h6giy3Yeqp3HCuz\nIi8i7PL4UNc5gs0l6ZrXbypOw+UV2VhflIp7b1yND1xSioxE1bvzkcvK8LNXzyy4zYtV3zWCNfna\nPaRzBStT0TvqnAlggBoKz0i0Bd1H9YYwihY39YxiVU7g38nW0oywh9qGHC788vWzUaknF4t6p+aH\nzg1gQOD9bhdr2KGK6NosJpRnJ+FUiHmdUkrc98SxmRWvy43HK2Gb6qXPT43HoMO1qAVNUkp0DU9i\ndV7ykt7LliGMlry1Balo6BqBx+8TUftQ4JWR01ZkJqKhaxR2S/CwJoTAF3dWBZwPBkyVCFjk3IR4\nmxmfu65C87rVYa6Q7B93zWyJ5C8vNW5Rb7DOEAVLA/lNbSvuqAk+VLypOE1zODgvNQ7JcRbN8h0X\ngqq/Fbxnc1qgMhXDDnfQsBWIySSwvig14Lwur0/CLIJvNWYxmyAhwwoXfz7Wier8FOw50YV/e+Yk\nfvn6WfQso1ICe0504Ya1gYe2r6rMjvpChicOteGdUz3sq3KScKo3+Ara3lEncpPjcDKCldDRMun2\n6jqH0O31wWKe/Vx9z5Zi/PZg64LPWdsyiJrSDKwvTMXRtuHFNtEwDGG0LNy8oQBPHz2/Aqlt0IGi\n9OBFb0szE/Db2tagQz7TrqvOXfR8DinlgoucFqbFo20odEHQQDmwIDU+KlsX+bOYBFyewEMB404P\nuoYnsConvN4kLX+7fQUe2X8u4OTbp4924H9eOa3L5NyekUnkaARaLYHKVLzY2IOrVy9sOPDmDQV4\n+kiH5nWHWwexqWR+j+dcF5dmYP+Z0JP82wYnUFOagQ9dWop/2rUa11fn4s/HOvH9ZxtD/mwHxl34\n5z8dx/f2NuBXb7SgrmNk0b1KUsqo/k7bhyYC1gwEgO0rM2b2u42G6Xmr03X1clPs6AoRak92jeJD\nl5ZGtR3hevJwB363iEAUSp3GB5rSrES0D00EfQ0J5oWTPbi6KhtrC1NxvIMhjMhQG4vTcLx9eObF\nv3UgdE9YYVo8XjvVjy0rtIfKIpWRaEPfWOD6V/4bd0cqnJ62Ub89BOfKSbGja4FV8wO9GRalx6M9\nSDD89f4W/O22FQt6zGlWswk3rc/Hk3PCiM8n8ZOXTsFiMmFtQSp+qtOwZbjBO1CZioauUVQFWPQR\nitkksDo/Bcfbz7/BOD1eNHSN4g9vteOSlaFXUl++Kivk3qOD4y6kzVm9l58aj49cVobr1+TiJy+d\nDnjfSbcX/++vjfjiDVX4ws4qXL8mF2f7x/GD55rw4O76kO0L5IlD7dhzIjobkasFDKFXuOakLK63\n2N/rp/qxvfz89IVwnkcnu0Zw6apM9I9f2Bp6Uko09YwiNd6q20rDg1OT8ue6fXMh/nioPeLzTQ/1\nWswmJNktcIQ5PzUW6RrChBC7hBANQohmIcS9Gtd/UghxTAhxWAjxqhCiWus8ROG4cX0+dh9TvWH9\n4y5kJmqvHJtmMZvwscvLZuYpLNbq/JSgQ4b+G3fr4Xj7CNYHWBgQZzUv+BNnz6gTOSnz644Fq+Qv\npcSQw73gXRP8bS3LQH3nKEam9kOccHnxrd31uHxVFnaty8Ml5Zkoz0nCr14/u+jHmub2zt76KRSt\nN1mXxweL2bSoHtTbNhXg5/vO4vt7G/D9vQ3475dPo6lnFB+9vCys4WGL2QSLKXj5lGfrunF9da7m\ndeuLUlGVl4TfHZy/jZjPJ/HdPQ341I7ymSHXvNQ43LQ+H/9wfSVKsxLRGkEtOX+N3WOoD6Pq//CE\nG//x16agt3m2rgvXV4deZXv75kL86XDkgUDLy019uKoish7QsUkPUuKsMAlxQeeFvdzUhysrs1Gc\nnhC17bfmCjTvdnVeCpp6RmdNJQnHn46049aNBdFqnqF0C2FCCDOAhwDcCKAawF0aIesRKeV6KeUm\nAN8B8H292kPL35YV6TjcOjTzAhbOm98nroreiqjVeclB3zjmbtwdqcwke9BK88fbhwOGMAALHgo9\n1aNdVqMkSMHWxu4xVOUtfBhyro9eXoafvnIGXcOTePAv9bj7ypWzFjBcXZWD7OQ4/LY2OkMqzT1j\nqFjEMCoAHDg7gG1loYe6g7GaTfjuezbi8zur8PmdVbjnmgrcvKEgoufRdWty8Vx94IKkZ/rHg57v\nmtW5cHt98+ZM/filU7j9osKApR0uK88Ku4yJv0m3F3FWVbIllANnBnCiYxj9QXqgz/Y7UBbGh5+M\nRBuGIpig3zMyOWtbM//jWUm2edvP2cOsJbgmPwX1ERamDmTI4cLwRPDN3F9r7sPlq7JQU5qB2hZ9\na6ZpefeWYnxr98mwt2aTUqJ9cPbwck6Kfcluh6RnT9hWAM1SytNSSheAxwDc5n8DKaX/My0Rqr4m\n0YLdsDYPe+uCryrTS1qCLegL3tyNuyMVavuiAYcrZGHRQMacnoArjE71jqE8Z/6bWFaSDX0BQmGo\n8gmRyk62Iy81Dj999TS+fNMazZ65XevyYLOY5g1dLsTcTbHDYTaZZi2Xf/PMALYuMoRFw7rCFBxr\n154zMzzhRnJc6BIgd20tweHWIdR1qJfsx2tbUV2QEnThQklmAloHIu9Z2XeqD5etykK8zRxyV4Vj\n7cP45jvX4bED2uF72BFZLaorKrLC3ibt92+1o75zBN94ug7P1nXPfPj7w6F2zZI3K7MScTrA5HzV\n86rejreWZWD/6eiEof99owU/eK4p4DBjQ5daYSuEQHl2Ik71LH4P1Lk6hoLXYKvKS8Y/7qrCs3Xd\n+NHzTRgL8Ts/1DqEzXOGNjcUpS3Zyfl6hrBCAP5/GW1Tx2YRQnxGCHEKqifss1onEkLcLYSoFULU\n9vZyKw4KbGtZBt48M7jolYoLFaqvaTFDU6vzkgNu3dQ/5pw3rycSfznWif988ZTmdb0Btn8SQgTs\nXRuaUNWxo+murSX4yjuqg9buum1TISbd3kVvRXOufzxoXTgtRenxaJ8azpFSwuOLjUKpQgikJ1g1\nt4p5/mQ3rlujPRQ5N5LMjgAAGK9JREFU1z1Xr8Lv32rD47WtkFLOVPkPJdJ5Rm+1DOGiknRcVJKO\nQ+eCr9jz+iRyUuLg9PgwOjn/A9Cz9d24vjq8dgLAJSsz8XqYE+NHJt1478Ul+Oo71iDJbsG3nzmJ\nR/afw/CEW3PorTwnCacCbF5/pm985oNOVpI9Ktv6eLw+jLu8uL46F08EmHf1pN+wnl6FZGtbBkPO\nu42zmvGxK1bi3VuK8aPnm/HEobaAQ7LP1Xfj2jnP2TX5wUchYpnhrxBSyoeklOUA/gnAVwPc5mEp\nZY2UsiY7m4UHKbjrqnOiHgDCZTKJwPMbFhkM0xJsGAnQ07b7eBduWh9839E4qylgXZ6z/ePITLQF\nnJgc8AVa43taaFmGaLmjphj7F7kVjQTmDSeFUpqViDNTQyqN3WMBd2Ewwk3r8/HnY/P3L2zsHkNF\nkFpj/kwmgS/dUAWLSeC9F5eEdZ+K3CQ0RbDvqc8nISFhNglsLErD4dbBgLf178V778XFeLx2/ry1\nU71jEa3OFUIgO8mOntHgQ1utA46ZlY9CCFxSnokv37QGW8sy8P7t2otRVmQmBNzovb5zBKv9hu9N\nAoueF/b8yR5cszoH21dmon1wYt78vN5RJ5LjrLM+1KQtYh/UQJq6R8N+juWlxuHeG1ejJCMBDzxd\nhwNnZ/8dj0y6EWcxz/twY7eYl2zRVj1DWDuAYr/LRVPHAnkMwDt1bA+9TVxanoWPXFZqyGNX5SZr\n1tuZdHujswAgQBhqG3QEXYIPqIKtwZbJ37WtBI9EuqebxiTiV5p7cYXGptIXUmaiLej8uWB8Phn2\nNlH+yrIS0TL1JvtCQw92VIbfA6O34owEtA3OfhMec3qQYDVH1AMSZzVHtEXYJeWZ2Bdidaa/w21D\n2FikhpribWZMugO/sR7wG+4tTItH/5hz1pyr0Uk3khaw28ItGwvw9JHgG24/W9eNnRqLGVblJM2E\ns7mCBYUzfeMozTw/5L86CvPCDrYMomaqB+ruq1bif145PWu+228PtuI9W2b/LresSMfBlsDBdyGk\njPwDzZYVGbj/5mp0Dk/im0/XzYTXp4504NZNgSfkL8V9JPUMYQcAVAghyoQQNgB3AnjS/wZCCP/K\nlO8AEHyZC1GYjNqj7Ya1eXjqSMe8lYgt/Q6URmFlpElg3sTh1oHQNdEAIC8lHp0aJSXaBh0oSItH\nSpwV8VbzrCKd404P4oNsdZSXEofuOb0G9Z0jqM6PbD5VtN2wNg97TixsbuC5AQdKMiP/XaUnWDHg\nUD2VIxOhN/6+0EozZ9cye+FkD65Zo29QzEmOQ99Y+D0rrzX14Qq/VYUWswgYXI61D89anPGuiwpn\nDbs9f7IH1y7g+8udek4He0PvHdNeMRxSgNcln0/C4te7s22R88I6hyeQmxI38zpot5hx17YS/Hzf\nWQDqQ+G40zNv2HRtQXRrbjlcngUVeQZUcLt1YwG+eEMVnjvZg39/thFnesexIsDfZnH6wuYgGk23\nECal9AC4B8AeAPUAHpdSnhBCPCCEuHXqZvcIIU4IIQ4D+DyAD+nVHqILwWQS+PBlZfjZa7PrVp3u\nHVvQtj9zlWQkzFuR+JfjnbgpjI2uC9K0ty7af3oA28pUTaP3bS3Bo2+en8p5pm8cK7MDBxJVpuJ8\ne1QvUvBK7hdCcUZC0BpmwSxkUj4wFfylRM/oZMg9Ao1w07p87D5+voen7gKF5aBD9HNMzNmZYW1B\nKk50aPcIeX1y1rDUqpz/396dR8dVX3cA/94ZSda+r7Y2y5IXGa/IKxRsIGaxizmlhz1QDiknCSnQ\nJmlIk0NPF9JmgRJayinNUtompkApuMSAHWMcUoKNwba8yLJleZFtrRbabdmSbv94T2I0mk2jmXlP\nmu/nHB9r3oxHP/npN3Pn9+7v3hScaOsd+V41jcHXaPP1fdt6+v2Wv/FGENhlxqwJ5oW9ue8cNrqt\nGM3NT0WMQ3DwbCfe3HcWGxeP3TwQF+PAwGDoVpP2N3SOrGwGKz7WiYeunokHVpfiruXeL4MvLErD\nfg+7Ve0urDlhqrpFVWer6ixVfco89qSqbja/fkxV56vqYlVdq6qHwjkeokiYmZ2E+BjHqETR+rbe\ngLbJ+zMnPwVH3BJQ23s9JwK7y0v1fDnyRFvvSHPutMRYxDhlpOis0bjbe/DoXivswNlOLCgMrNVP\nuGUmxvksXeBNXUuPz56M/rxXE9wKTLilJcai68IAVNVoTh4zvkuRwVowI83r7kxXJ9t6UZI1ekW3\nqiQDe06OXRHytqvzlgUF2HKwaaR3bLA/37rKPGw97LlY7K8PB76Zwd2M9ASccyua3Nl3GakeNtV4\nWvUOxNCQeq3L9cWVJXh1TwOO+chZjAuwlEYg9jV0YFFRaF4PMpPifM7L8pzx5R/aheWJ+URT0RdX\nlWLT7tMjl1L6B4aCXpZ3VZ6bjDqXF5ojTV2Ykx9YwBAf60S/hxdXhY56s7pneTE27TJyw06d7xvz\nxuiqwK0d0m/rjPICdrBufh62eXkj9aXv8kDQuxqdDgfq27xfMrHa4qJ07G3owM6jLbh2TmQ2Oa0o\nywxoo8T2Iy24bu7o4CYjKQ4dfWM3o+w56bn8x+KidFQ3dGBHrZGUHqz4WCeGhtRjgeNT7cGnFszK\nTRrTQ/JIUxfm5o9dkZxXkBrUjr/f1rXh98o9n1uHQ/DlNbN85vUtKvJd7mE8BXh7+wcitkknxumY\nlM3PGYQRhYHTIbh/VQleMnMwQsU9uXfroWasC6AauDeNnReQ75bbkpEUBxGj3cvAoO8yC07H6MT8\n3v6BoJKhw6EkKwmnxlmx3XhDDH5XY2FGAmKd1l6K9WXt3By8f6QF+890YlGEVixT42PRc9F37SfA\nKLPiqa2XcZV39Jurez6YqzVzcvHLXaeDuqTsau3cXOxwK3USbLL/sLLsZBx3W6050tSNuQVjf+eW\nzwwseHX34fHzWO3SMsldQVoCKn383ywtTsenpz0n558+34e/eetwQK2GhoY02PrQQQt29dBKDMKI\nwmR4a/yxZu8FVoNivrINDSkuXB702i/S178dtqu+HSvLxr5g3728GJvGuVOytbs/oMuikZSWML4t\n9+8ebMaN84MPapfPzMTt49g9GGnTYpwYUiBugu2UxsvfJS6jv6PnPKuynLGrRwOD6vXDwVXlWXj8\nhtkT/vmWFqePqVP2fu3EihBnmFX5XZ3v6feYQ5iVPM3j7+7gkOL5HXX43fGx9czaevqRkRg77t2I\nrlJ8BM2/3H0aP75rCQaHFG9V+y6KXN/W6zOVIRxm5Saj3kstNrtiEEYURn+0uhQv7DyOnOTQ1S0b\nrvf16enPcGXxxJqPe8t/ykqehsEhDfiNTFWx82gr1kToEleg1lXmYVtNYJckB4cU/QODSPSxG9Sf\noszECbWmioT1Cwtw84LgA81gXFmSgU99lD54z8dOzaqSzFH1ovxV+ReRkHQqEBFkJsWOyis83Bjc\npo1RxlFGwX1l5+LlQTz1qxqsq8xDc9dFPLO1dlRXgTf2nsVtHqr1j5enOmU1jV0ozUpEQpwTt19Z\niN7+Abxz0PsO5E8DKNIaagsL07F/klXOZxBGFEYxTge+umZWSPOkKnJTcKylGzuPtuKacX4q99S/\nzlugdc+KYqwNIK8mPTEOn/Vdxom2HpSFsUF5MMpyknHCS4FMdx/Vn8cqH5dxpop5Bakec5DCyV/9\nqRNt3jeAuHYiAIx8sGURage1YeF0vFVt7CjtHxhEbIhXEIeG1GdPV9e8sM4Ll/G9LTX442tmoiIv\nBbctmYH7VpbgmW1H8WFdG1QVrd39yAumdIab2fkpONoyegX/tU/O4HaXumJ3LivG+d5+/Nol71JV\nsa+hAz96txatPf0ozPDerigcSt02Ck0GDMKIwqw8NyWkqyNz81Nw8GwXVDHuArAFafEjVfFbui56\nzMEZlp08DYuL/G8vL81ORH1rD5xifWkKT1LjY/02MQaMRsarZ9ljU8FUEx/rRL+HJHfALGTs9L5p\nxf13qvqMn0b1ITTdZTfjh3XncVUIgvTUhFh0mpsNGj7r89kea0VZFnadaEdT50U8vbUWX//CnFF9\nGHNT4/Hd9fPQ1nsJ33i1OmS9So3Vx8+D5t0n2rGkOH3MJeB7V5TgbMcFbDnQiP/46BT+/u0jONdx\nAY9eX4FH1pZH/PXAjq8//jAII5pkZqQn4JU9DUFVpc9P+7xW2Ecn2rGibOIv2sWZSXh971ksjfCl\nh0B9oTIX2/1ckrxwyeho4JxALg35lhwfgy4P/R23HGjEVeW+g5u81GkjHx7c64OF27x8YzVqz6l2\nVJVOfL7Myk3G8TYjb6mmsdvnRpDMpDgcberGi78xGtd7KgAsYhQ1fXJDZcD9PP3JT4sfKdqsqthy\noBG3XOG5LdoDq0sR4xBcW5GDb98yD7csKAhNd5AgxTgcHne12hWDMKJJxuEQlGUnYVkQbwhGSQnj\nk/2x5m7MHkdfPW+KMhOw9VCzxwR/OyjPTfFbP2jr4aYJ7TIl/1bMzMRulyrwPf0DeGZrLQD4zR2q\nKjXywvzlg4XDjfPz8bbZdzMUQXp5zuc7JOtauv3WpLtnRTG+s36ez8b1gFEHbiIJ+d5sN+ve+Xru\ndfPzUeyjlE0kzclPQW1TiDdDhRGDMKJJ6Ok7FgX1glvgshIWTE83T6bFOHHfymK/bxJWSp4Wg24P\nqzDDahq7fW7bp4lbMCMN1WbR1g+OteLZbUdx36oS/MHSQr+XkebkpeBoc3dE88GGJcQ50d53KWRJ\n5tPTE0a6OVwaGPI7bxYVpVuyQluQloCG9j783/E2XG2T2n+BWORWOb+99xL+d/85/PP7dRaOyjt7\nFPQhonEJNvdhODfHKCcRuh2bj98wO2TPFQ7Xz8vFe0daPLZqMdoMhe7/gjyLcTrQf3kQ33/nCK6Y\nnobvrJ8X8O+xwyEYUkX1mU587bryMI90rMdvmI00D1Xtg+F0CCZDKatlpRl48s2DeCwE5T4iKT81\nHntPd6C99xL6BwaRkRiHq8qzcb6nH81dF0OycSGUGIQRRaHdJz7vFxkN5uSlYNOu01hYmD6mfdSv\nqhuxYeF0L/+SQun3F03H9PQEZAbRezElPhZNnRcjmg82LBy9QPsuDSA+BF00wmVWTjIqp6cGtDnH\nTkQEX107C9PTEkZ1KUmIdWJnbSvuWFZk4ejG4uVIoig00crwk42I4LsbKrHlQOOYat/NXf3IT7PX\np+Op6ooZaUEFYICRN5abaq9iwMGKdQgOn+sKusF4JDgcgm/eONfqYQRlVk7ymDZxpdlJOGnD8hUM\nwoiiUKjywSaTWKcDj6wtR3byNPzdlhr09A/gWLP/xGiyh6qSDHzp6jKrhxESxVmJePdQE+ZE0Qch\nO4hxOka1fbMDBmFEUWZajAMZQa5GTAVXV2Tj4WvK8PTWWryw8zhuuoK7IicDEbG09EEozcpJxgfH\n2jAjPbLFTKPd0uJ0n0WDrTA1fqOJKGAVucmTardTOGQlT8OTGypx/6pS2zQcp+hRlpOE1PjYSZXw\nPhWsLMvCrvrxN0UPJ776EEWZdRNoUD2ViMikSzqmqSExLgaPWLDLM9oZu8O9N5K3AlfCiIiIIuza\ncfZ9pdAwuoZc8P/ACGEQRkRERFFhzexc7KxttXoYIxiEERERUVQozkrE6fY+q4cxgkEYERERRY0Y\np32afDMIIyIioqhRVZJhm1IVDMKIiIgoaiyfmYndJ+xRqoJBGBEREUUNO5WqYBBGREREUaUgPQHn\nOqwvVcEgjIiIiKLKmtk52HnU+lIVDMKIiIgoqhRlJuLMZ9aXqmAQRkRERFGnODMRqmrpGNg7koiI\niKLOncuKrR4CV8KIiIiIrMAgjIiIiMgCDMKIiIiILMAgjIiIiMgCDMKIiIiILMAgjIiIiMgCDMKI\niIiILMAgjIiIiMgCYQ3CROQmEakVkToRecLD/X8mIodFpFpEtotISTjHQ0RERGQXYQvCRMQJ4HkA\nNwOoBHC3iFS6PWwvgCpVXQjgNQA/CNd4iIiIiOwknCthywHUqWq9ql4C8DKAja4PUNUdqjrcQfMj\nAIVhHA8RERGRbYQzCJsBoMHl9hnzmDcPAXjb0x0i8rCI7BGRPa2trSEcIhEREZE1bJGYLyL3AagC\n8ENP96vqi6papapVOTk5kR0cERERURjEhPG5zwIocrldaB4bRURuAPAdANeqan8Yx0NERERkG+Fc\nCfsYQIWIzBSROAB3Adjs+gARWQLgXwDcqqotYRwLERERka2ELQhT1QEAXwPwLoAaAK+o6iER+WsR\nudV82A8BJAN4VUT2ichmL09HRERENKWIqlo9hnERkVYAp8L8bbIBtIX5e1BweG7siefFvnhu7Inn\nxb5CfW5KVNVjQvukC8IiQUT2qGqV1eOgsXhu7Innxb54buyJ58W+InlubLE7koiIiCjaMAgjIiIi\nsgCDMM9etHoA5BXPjT3xvNgXz4098bzYV8TODXPCiIiIiCzAlTAiIiIiCzAIIyIiIrIAgzA3InKT\niNSKSJ2IPGH1eKKViBSJyA4ROSwih0TkMfN4pohsE5Fj5t8ZVo81WomIU0T2ishb5u2ZIrLLnDv/\nZXbKoAgSkXQReU1EjohIjYis4pyxBxH5U/O17KCIbBKReM4Za4jIz0SkRUQOuhzzOE/E8Jx5jqpF\nZGkox8IgzIWIOAE8D+BmAJUA7haRSmtHFbUGAHxdVSsBrATwiHkungCwXVUrAGw3b5M1HoPRDWPY\n9wH8g6qWA/gMwEOWjCq6/RjAO6o6F8AiGOeHc8ZiIjIDwKMAqlT1CgBOGK38OGes8W8AbnI75m2e\n3AygwvzzMIAXQjkQBmGjLQdQp6r1qnoJwMsANlo8pqikqo2q+qn5dTeMN5MZMM7HS+bDXgJwmzUj\njG4iUghgPYCfmLcFwHUAXjMfwnMTYSKSBuAaAD8FAFW9pKod4JyxixgACSISAyARQCM4Zyyhqr8B\n0O522Ns82Qjg39XwEYB0ESkI1VgYhI02A0CDy+0z5jGykIiUAlgCYBeAPFVtNO9qApBn0bCi3bMA\n/hzAkHk7C0CH2TMW4NyxwkwArQB+bl4m/omIJIFzxnKqehbAjwCchhF8dQL4BJwzduJtnoQ1LmAQ\nRrYmIskA/hvA46ra5XqfGvVVWGMlwkRkA4AWVf3E6rHQKDEAlgJ4QVWXAOiF26VHzhlrmPlFG2EE\nytMBJGHs5TCyiUjOEwZho50FUORyu9A8RhYQkVgYAdgvVPV183Dz8FKw+XeLVeOLYlcBuFVETsK4\nZH8djFykdPNSC8C5Y4UzAM6o6i7z9mswgjLOGevdAOCEqraq6mUAr8OYR5wz9uFtnoQ1LmAQNtrH\nACrMHStxMBInN1s8pqhk5hj9FECNqj7jctdmAA+YXz8A4M1Ijy3aqeq3VbVQVUthzJH3VPVeADsA\n/KH5MJ6bCFPVJgANIjLHPHQ9gMPgnLGD0wBWikii+do2fG44Z+zD2zzZDOB+c5fkSgCdLpctJ4wV\n892IyC0w8l2cAH6mqk9ZPKSoJCJXA/gAwAF8nnf0FzDywl4BUAzgFIA7VNU9wZIiRETWAPiGqm4Q\nkTIYK2OZAPYCuE9V+60cX7QRkcUwNkvEAagH8CCMD9ucMxYTkb8CcCeMnd97AXwJRm4R50yEicgm\nAGsAZANoBvCXAN6Ah3liBs3/BOPycR+AB1V1T8jGwiCMiIiIKPJ4OZKIiIjIAgzCiIiIiCzAIIyI\niIjIAgzCiIiIiCzAIIyIiIjIAgzCiGjSE5FBEdnn8idkTapFpFREDobq+YiIhsX4fwgRke1dUNXF\nVg+CiGg8uBJGRFOWiJwUkR+IyAER2S0i5ebxUhF5T0SqRWS7iBSbx/NE5H9EZL/5Z7X5VE4R+VcR\nOSQiW0UkwXz8oyJy2Hyely36MYlokmIQRkRTQYLb5cg7Xe7rVNUFMKpeP2se+0cAL6nqQgC/APCc\nefw5ADtVdRGMvouHzOMVAJ5X1fkAOgDcbh5/AsAS83m+HK4fjoimJlbMJ6JJT0R6VDXZw/GTAK5T\n1XqzIXyTqmaJSBuAAlW9bB5vVNVsEWkFUOjaOkZESgFsU9UK8/a3AMSq6t+KyDsAemC0PHlDVXvC\n/KMS0RTClTAimurUy9fj4drPbxCf59OuB/A8jFWzj0WEebZEFDAGYUQ01d3p8vfvzK8/BHCX+fW9\nMJrFA8B2AF8BABFxikiatycVEQeAIlXdAeBbANIAjFmNIyLyhp/aiGgqSBCRfS6331HV4TIVGSJS\nDWM1627z2J8A+LmIfBNAK4AHzeOPAXhRRB6CseL1FQCNXr6nE8B/moGaAHhOVTtC9hMR0ZTHnDAi\nmrLMnLAqVW2zeixERO54OZKIiIjIAlwJIyIiIrIAV8KIiIiILMAgjIiIiMgCDMKIiIiILMAgjIiI\niMgCDMKIiIiILPD/IRohn1+zhdkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_loss(m1, figsize=(10,7), downsample=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u2VWYVWNR0_K"
   },
   "source": [
    "# With sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lthUbpn6R5mm"
   },
   "outputs": [],
   "source": [
    "X = np.vstack((np.load(\"data/sequenced_data_for_VAE_length-160_stride-10_pt1.npy\"),\n",
    "               np.load(\"data/sequenced_data_for_VAE_length-160_stride-10_pt2.npy\")))\n",
    "y = np.load(\"data/sequenced_data_for_VAE_length-160_stride-10_targets.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kSbBqmZrSFuu"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=0.2, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uP7mTPvqSF5x"
   },
   "outputs": [],
   "source": [
    "scaler = RobustScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1o58gaXd8Jlf"
   },
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LYKHTsaRSF3_"
   },
   "outputs": [],
   "source": [
    "train = d.H_alphaSequences(X_train, y_train)\n",
    "valid = d.H_alphaSequences(X_valid, y_valid)\n",
    "test = d.H_alphaSequences(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7jGGuGd3SF2K"
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset = train, batch_size=512, shuffle=True)\n",
    "valid_loader = torch.utils.data.DataLoader(dataset = valid, batch_size=512, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Hddr9EK9RNkH"
   },
   "source": [
    "## Loading Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sKomVcQoPRFq"
   },
   "outputs": [],
   "source": [
    "VAE = torch.load(\"models_and_losses/CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "colab_type": "code",
    "id": "3TJfsR3VRbN9",
    "outputId": "43c81845-f017-4432-825c-a959f587eb93"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Reshape()\n",
       "  (1): Conv1d(1, 16, kernel_size=(6,), stride=(2,), bias=False)\n",
       "  (2): ReLU()\n",
       "  (3): Conv1d(16, 32, kernel_size=(4,), stride=(2,), bias=False)\n",
       "  (4): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (5): ReLU()\n",
       "  (6): Conv1d(32, 64, kernel_size=(4,), stride=(2,), bias=False)\n",
       "  (7): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (8): ReLU()\n",
       "  (9): Flatten()\n",
       "  (10): Linear(in_features=1152, out_features=256, bias=True)\n",
       "  (11): ReLU()\n",
       "  (12): VariationalLayer(\n",
       "    (mu): Linear(in_features=256, out_features=15, bias=True)\n",
       "    (rho): Linear(in_features=256, out_features=15, bias=True)\n",
       "    (softplus): Softplus(beta=1, threshold=20)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VAE.encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HQrz4pcRRvZC"
   },
   "source": [
    "## Freezing Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FepPrHmi5kKh"
   },
   "outputs": [],
   "source": [
    "class Encoder_Classifier(nn.Module):\n",
    "    def __init__(self, encoder, clf):\n",
    "        super(Encoder_Classifier, self).__init__()\n",
    "        self.encoder=encoder\n",
    "        self.clf = clf\n",
    "        self.frozen_encoder = False\n",
    "\n",
    "    def freeze_encoder(self):\n",
    "        for param in self.encoder.parameters():\n",
    "            param.requires_grad = False\n",
    "        self.frozen_encoder = True\n",
    "        print(\"Encoder frozen.\")\n",
    "\n",
    "    def unfreeze_encoder(self):\n",
    "        for param in self.encoder.parameters():\n",
    "            param.requires_grad = True\n",
    "        self.frozen_encoder = False\n",
    "        print(\"Encoder unfrozen.\")\n",
    "    \n",
    "    def forward(self, X):\n",
    "        Z, mu, sigma = self.encoder(X)\n",
    "        return self.clf(Z)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f0r4jLHe5tPy"
   },
   "outputs": [],
   "source": [
    "DNN1 = nn.Sequential(\n",
    "            nn.Linear(in_features=15, out_features=64),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(in_features=64, out_features=128),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(in_features=128, out_features=4),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=15, out_features=64, bias=True)\n",
       "  (1): Sigmoid()\n",
       "  (2): Linear(in_features=64, out_features=128, bias=True)\n",
       "  (3): Sigmoid()\n",
       "  (4): Linear(in_features=128, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DNN1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "-sou9uan7bFD",
    "outputId": "473dc211-21a7-4a1c-f9b7-1a9418fcc45e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder frozen.\n"
     ]
    }
   ],
   "source": [
    "model = Encoder_Classifier(encoder=VAE.encoder, clf = DNN1)\n",
    "model.freeze_encoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xw7AbVFD7TKP"
   },
   "outputs": [],
   "source": [
    "optimizer= torch.optim.Adam(model.parameters(), lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "hfD_WT5J7Yb-",
    "outputId": "0803e7a8-0aab-455e-9878-315ed7f840ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "m1 = Trainer(\n",
    "        model=model,\n",
    "        optimizer=optimizer,\n",
    "        loss_function=nn.CrossEntropyLoss(),\n",
    "        scheduler=torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30], gamma=0.1),\n",
    "        tensorboard=True,\n",
    "        model_name=\"DNN_on_CNN-VAE_sampling_latent_space_m.IV_lr-{1e-2,1e-3}_epochs-512\",\n",
    "        verbose=True\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mNFqptqd7Y_2",
    "outputId": "98abb786-b97f-40f7-81b2-40ce2d0e9a86"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n",
      "\t%eps : Float(512, 15) = aten::randn_like(%sigma, %133, %134, %135, %136), scope: Encoder_Classifier/Sequential[encoder]/VariationalLayer[12] # Z:\\Semi-supervised-Learning-with-VAE\\utils\\layers.py:35:0\n",
      "This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace()\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[286, 1] (1.698385238647461 vs. 1.635845422744751) and 2040 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[511, 0] (0.6252045631408691 vs. 0.7613823413848877) and 2042 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[346, 0] (1.1761618852615356 vs. 1.6192175149917603) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[209, 0] (1.0876561403274536 vs. 1.6349793672561646) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[295, 1] (0.8871971368789673 vs. 0.2903026044368744) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[143, 2] (-1.6825700998306274 vs. -0.8990321159362793) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[270, 3] (0.3332168459892273 vs. -0.7593979835510254) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[298, 2] (-1.6193972826004028 vs. -0.2547914385795593) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[471, 2] (-1.1326136589050293 vs. -3.108886480331421) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[365, 2] (-2.6952297687530518 vs. -5.257594585418701) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], average_loss:0.7864, validation_loss:0.6088, val_accuracy:0.7532\n",
      "Epoch [2/100], average_loss:0.5899, validation_loss:0.5435, val_accuracy:0.7922\n",
      "Epoch [3/100], average_loss:0.5338, validation_loss:0.5159, val_accuracy:0.8701\n",
      "Epoch [4/100], average_loss:0.5076, validation_loss:0.4949, val_accuracy:0.8182\n",
      "Epoch [5/100], average_loss:0.4904, validation_loss:0.4821, val_accuracy:0.9221\n",
      "Epoch [6/100], average_loss:0.4726, validation_loss:0.4593, val_accuracy:0.8442\n",
      "Epoch [7/100], average_loss:0.4522, validation_loss:0.4407, val_accuracy:0.9091\n",
      "Epoch [8/100], average_loss:0.4351, validation_loss:0.4270, val_accuracy:0.9091\n",
      "Epoch [9/100], average_loss:0.4229, validation_loss:0.4183, val_accuracy:0.8442\n",
      "Epoch [10/100], average_loss:0.4169, validation_loss:0.4194, val_accuracy:0.8312\n",
      "Epoch [11/100], average_loss:0.4102, validation_loss:0.4117, val_accuracy:0.7922\n",
      "Epoch [12/100], average_loss:0.4091, validation_loss:0.4107, val_accuracy:0.8312\n",
      "Epoch [13/100], average_loss:0.3995, validation_loss:0.4032, val_accuracy:0.8571\n",
      "Epoch [14/100], average_loss:0.3968, validation_loss:0.4030, val_accuracy:0.8182\n",
      "Epoch [15/100], average_loss:0.3934, validation_loss:0.4175, val_accuracy:0.8182\n",
      "Epoch [16/100], average_loss:0.3904, validation_loss:0.4001, val_accuracy:0.8312\n",
      "Epoch [17/100], average_loss:0.3897, validation_loss:0.3867, val_accuracy:0.8182\n",
      "Epoch [18/100], average_loss:0.3875, validation_loss:0.3980, val_accuracy:0.8442\n",
      "Epoch [19/100], average_loss:0.3826, validation_loss:0.3836, val_accuracy:0.8831\n",
      "Epoch [20/100], average_loss:0.3792, validation_loss:0.3872, val_accuracy:0.8442\n",
      "Epoch [21/100], average_loss:0.3787, validation_loss:0.3859, val_accuracy:0.8831\n",
      "Epoch [22/100], average_loss:0.3747, validation_loss:0.3886, val_accuracy:0.8442\n",
      "Epoch [23/100], average_loss:0.3688, validation_loss:0.3931, val_accuracy:0.7662\n",
      "Epoch [24/100], average_loss:0.3697, validation_loss:0.3874, val_accuracy:0.8701\n",
      "Epoch [25/100], average_loss:0.3682, validation_loss:0.3814, val_accuracy:0.7922\n",
      "Epoch [26/100], average_loss:0.3690, validation_loss:0.3734, val_accuracy:0.9221\n",
      "Epoch [27/100], average_loss:0.3635, validation_loss:0.3807, val_accuracy:0.8182\n",
      "Epoch [28/100], average_loss:0.3621, validation_loss:0.3850, val_accuracy:0.8312\n",
      "Epoch [29/100], average_loss:0.3635, validation_loss:0.3786, val_accuracy:0.9221\n",
      "Epoch [30/100], average_loss:0.3619, validation_loss:0.3875, val_accuracy:0.7922\n",
      "Epoch [31/100], average_loss:0.3510, validation_loss:0.3764, val_accuracy:0.7792\n",
      "Epoch [32/100], average_loss:0.3484, validation_loss:0.3739, val_accuracy:0.8182\n",
      "Epoch [33/100], average_loss:0.3468, validation_loss:0.3726, val_accuracy:0.8571\n",
      "Epoch [34/100], average_loss:0.3469, validation_loss:0.3752, val_accuracy:0.8442\n",
      "Epoch [35/100], average_loss:0.3468, validation_loss:0.3695, val_accuracy:0.8442\n",
      "Epoch [36/100], average_loss:0.3453, validation_loss:0.3713, val_accuracy:0.8571\n",
      "Epoch [37/100], average_loss:0.3476, validation_loss:0.3675, val_accuracy:0.9221\n",
      "Epoch [38/100], average_loss:0.3463, validation_loss:0.3712, val_accuracy:0.8831\n",
      "Epoch [39/100], average_loss:0.3474, validation_loss:0.3768, val_accuracy:0.7662\n",
      "Epoch [40/100], average_loss:0.3461, validation_loss:0.3699, val_accuracy:0.8701\n",
      "Epoch [41/100], average_loss:0.3464, validation_loss:0.3729, val_accuracy:0.8052\n",
      "Epoch [42/100], average_loss:0.3453, validation_loss:0.3718, val_accuracy:0.7922\n",
      "Epoch [43/100], average_loss:0.3445, validation_loss:0.3722, val_accuracy:0.7662\n",
      "Epoch [44/100], average_loss:0.3444, validation_loss:0.3719, val_accuracy:0.7662\n",
      "Epoch [45/100], average_loss:0.3446, validation_loss:0.3672, val_accuracy:0.9221\n",
      "Epoch [46/100], average_loss:0.3437, validation_loss:0.3678, val_accuracy:0.8831\n",
      "Epoch [47/100], average_loss:0.3451, validation_loss:0.3734, val_accuracy:0.8571\n",
      "Epoch [48/100], average_loss:0.3430, validation_loss:0.3679, val_accuracy:0.8701\n",
      "Epoch [49/100], average_loss:0.3443, validation_loss:0.3727, val_accuracy:0.8571\n",
      "Epoch [50/100], average_loss:0.3431, validation_loss:0.3699, val_accuracy:0.8571\n",
      "Epoch [51/100], average_loss:0.3435, validation_loss:0.3690, val_accuracy:0.8312\n",
      "Epoch [52/100], average_loss:0.3445, validation_loss:0.3668, val_accuracy:0.8701\n",
      "Epoch [53/100], average_loss:0.3425, validation_loss:0.3648, val_accuracy:0.8831\n",
      "Epoch [54/100], average_loss:0.3431, validation_loss:0.3685, val_accuracy:0.8831\n",
      "Epoch [55/100], average_loss:0.3436, validation_loss:0.3701, val_accuracy:0.8442\n",
      "Epoch [56/100], average_loss:0.3435, validation_loss:0.3753, val_accuracy:0.8182\n",
      "Epoch [57/100], average_loss:0.3429, validation_loss:0.3676, val_accuracy:0.8831\n",
      "Epoch [58/100], average_loss:0.3438, validation_loss:0.3684, val_accuracy:0.8701\n",
      "Epoch [59/100], average_loss:0.3410, validation_loss:0.3692, val_accuracy:0.8961\n",
      "Epoch [60/100], average_loss:0.3415, validation_loss:0.3716, val_accuracy:0.8052\n",
      "Epoch [61/100], average_loss:0.3413, validation_loss:0.3679, val_accuracy:0.8052\n",
      "Epoch [62/100], average_loss:0.3433, validation_loss:0.3715, val_accuracy:0.7662\n",
      "Epoch [63/100], average_loss:0.3408, validation_loss:0.3675, val_accuracy:0.8442\n",
      "Epoch [64/100], average_loss:0.3412, validation_loss:0.3660, val_accuracy:0.8961\n",
      "Epoch [65/100], average_loss:0.3418, validation_loss:0.3666, val_accuracy:0.8831\n",
      "Epoch [66/100], average_loss:0.3430, validation_loss:0.3741, val_accuracy:0.8052\n",
      "Epoch [67/100], average_loss:0.3429, validation_loss:0.3637, val_accuracy:0.9091\n",
      "Epoch [68/100], average_loss:0.3399, validation_loss:0.3702, val_accuracy:0.8701\n",
      "Epoch [69/100], average_loss:0.3422, validation_loss:0.3726, val_accuracy:0.8571\n",
      "Epoch [70/100], average_loss:0.3404, validation_loss:0.3639, val_accuracy:0.8831\n",
      "Epoch [71/100], average_loss:0.3412, validation_loss:0.3722, val_accuracy:0.8312\n",
      "Epoch [72/100], average_loss:0.3397, validation_loss:0.3676, val_accuracy:0.8961\n",
      "Epoch [73/100], average_loss:0.3396, validation_loss:0.3674, val_accuracy:0.8571\n",
      "Epoch [74/100], average_loss:0.3412, validation_loss:0.3732, val_accuracy:0.8312\n",
      "Epoch [75/100], average_loss:0.3412, validation_loss:0.3753, val_accuracy:0.7792\n",
      "Epoch [76/100], average_loss:0.3398, validation_loss:0.3640, val_accuracy:0.8961\n",
      "Epoch [77/100], average_loss:0.3401, validation_loss:0.3689, val_accuracy:0.8571\n",
      "Epoch [78/100], average_loss:0.3387, validation_loss:0.3682, val_accuracy:0.8312\n",
      "Epoch [79/100], average_loss:0.3377, validation_loss:0.3678, val_accuracy:0.8442\n",
      "Epoch [80/100], average_loss:0.3382, validation_loss:0.3656, val_accuracy:0.8831\n",
      "Epoch [81/100], average_loss:0.3376, validation_loss:0.3664, val_accuracy:0.8701\n",
      "Epoch [82/100], average_loss:0.3377, validation_loss:0.3660, val_accuracy:0.8571\n",
      "Epoch [83/100], average_loss:0.3382, validation_loss:0.3696, val_accuracy:0.8571\n",
      "Epoch [84/100], average_loss:0.3381, validation_loss:0.3709, val_accuracy:0.8961\n",
      "Epoch [85/100], average_loss:0.3376, validation_loss:0.3679, val_accuracy:0.8312\n",
      "Epoch [86/100], average_loss:0.3398, validation_loss:0.3670, val_accuracy:0.8831\n",
      "Epoch [87/100], average_loss:0.3372, validation_loss:0.3633, val_accuracy:0.8571\n",
      "Epoch [88/100], average_loss:0.3378, validation_loss:0.3673, val_accuracy:0.8442\n",
      "Epoch [89/100], average_loss:0.3378, validation_loss:0.3656, val_accuracy:0.9091\n",
      "Epoch [90/100], average_loss:0.3372, validation_loss:0.3712, val_accuracy:0.7792\n",
      "Epoch [91/100], average_loss:0.3369, validation_loss:0.3741, val_accuracy:0.7792\n",
      "Epoch [92/100], average_loss:0.3360, validation_loss:0.3701, val_accuracy:0.8182\n",
      "Epoch [93/100], average_loss:0.3374, validation_loss:0.3720, val_accuracy:0.8312\n",
      "Epoch [94/100], average_loss:0.3368, validation_loss:0.3707, val_accuracy:0.8961\n",
      "Epoch [95/100], average_loss:0.3385, validation_loss:0.3745, val_accuracy:0.7403\n",
      "Epoch [96/100], average_loss:0.3366, validation_loss:0.3709, val_accuracy:0.8052\n",
      "Epoch [97/100], average_loss:0.3364, validation_loss:0.3665, val_accuracy:0.8312\n",
      "Epoch [98/100], average_loss:0.3352, validation_loss:0.3704, val_accuracy:0.7532\n",
      "Epoch [99/100], average_loss:0.3366, validation_loss:0.3694, val_accuracy:0.8052\n",
      "Epoch [100/100], average_loss:0.3361, validation_loss:0.3680, val_accuracy:0.8442\n"
     ]
    }
   ],
   "source": [
    "lh = m1(epochs=100, train_loader=train_loader, validation_loader=valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5_qgCvyEN8N-"
   },
   "outputs": [],
   "source": [
    "X_test = np.load(\"data/encoded_data_test_CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300.npy\")\n",
    "y_test = np.load(\"data/labels_test.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dl-v7vuW7ZGJ"
   },
   "outputs": [],
   "source": [
    "#m1.tb.close()\n",
    "\n",
    "m1.model.eval()\n",
    "with torch.no_grad():\n",
    "    x_pred = np.argmax(m1.model.clf(torch.tensor(X_test).float().to(m1.device)).cpu().detach(), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "nVJcVBba7ZQj",
    "outputId": "ad9c9adf-e291-4a61-a997-f61280318e43"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8318463772913665"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred=x_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "UpfUu-pxOFiW",
    "outputId": "93e7f8c2-3088-4f86-fd14-ba7e644eb74d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8589866156787763"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "TBNwia05OFl4",
    "outputId": "bcfd8546-69ad-44f6-8f01-f3451a830d2f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3102, 1002,   22,  109],\n",
       "       [ 683, 9639,   26,   97],\n",
       "       [  20,   33,  403,   23],\n",
       "       [ 204,  130,   11, 1232]], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EjnpeT8y7ZEE"
   },
   "outputs": [],
   "source": [
    "#torch.save(m1.model.clf, \"models_and_losses/CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300+DNN_on_CNN-VAE_sampling_latent_space_m.IV_lr-{1e-2,1e-3}_bs-512_epoch-100.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [101/200], average_loss:0.3372, validation_loss:0.3726, val_accuracy:0.8052\n",
      "Epoch [102/200], average_loss:0.3360, validation_loss:0.3708, val_accuracy:0.8052\n",
      "Epoch [103/200], average_loss:0.3364, validation_loss:0.3681, val_accuracy:0.8052\n",
      "Epoch [104/200], average_loss:0.3350, validation_loss:0.3661, val_accuracy:0.8312\n",
      "Epoch [105/200], average_loss:0.3363, validation_loss:0.3664, val_accuracy:0.9091\n",
      "Epoch [106/200], average_loss:0.3351, validation_loss:0.3661, val_accuracy:0.8571\n",
      "Epoch [107/200], average_loss:0.3354, validation_loss:0.3679, val_accuracy:0.8312\n",
      "Epoch [108/200], average_loss:0.3345, validation_loss:0.3681, val_accuracy:0.8701\n",
      "Epoch [109/200], average_loss:0.3354, validation_loss:0.3718, val_accuracy:0.8831\n",
      "Epoch [110/200], average_loss:0.3348, validation_loss:0.3673, val_accuracy:0.8182\n",
      "Epoch [111/200], average_loss:0.3344, validation_loss:0.3720, val_accuracy:0.7922\n",
      "Epoch [112/200], average_loss:0.3345, validation_loss:0.3665, val_accuracy:0.8701\n",
      "Epoch [113/200], average_loss:0.3347, validation_loss:0.3721, val_accuracy:0.8182\n",
      "Epoch [114/200], average_loss:0.3334, validation_loss:0.3639, val_accuracy:0.8701\n",
      "Epoch [115/200], average_loss:0.3344, validation_loss:0.3681, val_accuracy:0.8052\n",
      "Epoch [116/200], average_loss:0.3344, validation_loss:0.3685, val_accuracy:0.8571\n",
      "Epoch [117/200], average_loss:0.3333, validation_loss:0.3667, val_accuracy:0.9351\n",
      "Epoch [118/200], average_loss:0.3349, validation_loss:0.3680, val_accuracy:0.8442\n",
      "Epoch [119/200], average_loss:0.3335, validation_loss:0.3655, val_accuracy:0.9091\n",
      "Epoch [120/200], average_loss:0.3333, validation_loss:0.3643, val_accuracy:0.8701\n",
      "Epoch [121/200], average_loss:0.3335, validation_loss:0.3661, val_accuracy:0.8182\n",
      "Epoch [122/200], average_loss:0.3335, validation_loss:0.3734, val_accuracy:0.8571\n",
      "Epoch [123/200], average_loss:0.3326, validation_loss:0.3675, val_accuracy:0.8701\n",
      "Epoch [124/200], average_loss:0.3320, validation_loss:0.3730, val_accuracy:0.8182\n",
      "Epoch [125/200], average_loss:0.3311, validation_loss:0.3741, val_accuracy:0.8442\n",
      "Epoch [126/200], average_loss:0.3330, validation_loss:0.3696, val_accuracy:0.8052\n",
      "Epoch [127/200], average_loss:0.3330, validation_loss:0.3675, val_accuracy:0.8701\n",
      "Epoch [128/200], average_loss:0.3336, validation_loss:0.3653, val_accuracy:0.8831\n",
      "Epoch [129/200], average_loss:0.3332, validation_loss:0.3732, val_accuracy:0.7662\n",
      "Epoch [130/200], average_loss:0.3320, validation_loss:0.3658, val_accuracy:0.9091\n",
      "Epoch [131/200], average_loss:0.3324, validation_loss:0.3672, val_accuracy:0.8442\n",
      "Epoch [132/200], average_loss:0.3309, validation_loss:0.3664, val_accuracy:0.8312\n",
      "Epoch [133/200], average_loss:0.3321, validation_loss:0.3713, val_accuracy:0.8442\n",
      "Epoch [134/200], average_loss:0.3315, validation_loss:0.3698, val_accuracy:0.8442\n",
      "Epoch [135/200], average_loss:0.3322, validation_loss:0.3737, val_accuracy:0.7922\n",
      "Epoch [136/200], average_loss:0.3327, validation_loss:0.3668, val_accuracy:0.8961\n",
      "Epoch [137/200], average_loss:0.3325, validation_loss:0.3650, val_accuracy:0.8701\n",
      "Epoch [138/200], average_loss:0.3319, validation_loss:0.3692, val_accuracy:0.8701\n",
      "Epoch [139/200], average_loss:0.3320, validation_loss:0.3730, val_accuracy:0.8182\n",
      "Epoch [140/200], average_loss:0.3316, validation_loss:0.3660, val_accuracy:0.8701\n",
      "Epoch [141/200], average_loss:0.3302, validation_loss:0.3730, val_accuracy:0.8182\n",
      "Epoch [142/200], average_loss:0.3319, validation_loss:0.3699, val_accuracy:0.8052\n",
      "Epoch [143/200], average_loss:0.3311, validation_loss:0.3652, val_accuracy:0.8571\n",
      "Epoch [144/200], average_loss:0.3315, validation_loss:0.3656, val_accuracy:0.8442\n",
      "Epoch [145/200], average_loss:0.3290, validation_loss:0.3645, val_accuracy:0.8571\n",
      "Epoch [146/200], average_loss:0.3284, validation_loss:0.3679, val_accuracy:0.8182\n",
      "Epoch [147/200], average_loss:0.3303, validation_loss:0.3675, val_accuracy:0.8961\n",
      "Epoch [148/200], average_loss:0.3299, validation_loss:0.3707, val_accuracy:0.8052\n",
      "Epoch [149/200], average_loss:0.3294, validation_loss:0.3688, val_accuracy:0.8831\n",
      "Epoch [150/200], average_loss:0.3309, validation_loss:0.3659, val_accuracy:0.8571\n",
      "Epoch [151/200], average_loss:0.3291, validation_loss:0.3642, val_accuracy:0.8831\n",
      "Epoch [152/200], average_loss:0.3300, validation_loss:0.3685, val_accuracy:0.8312\n",
      "Epoch [153/200], average_loss:0.3300, validation_loss:0.3741, val_accuracy:0.7273\n",
      "Epoch [154/200], average_loss:0.3295, validation_loss:0.3663, val_accuracy:0.9091\n",
      "Epoch [155/200], average_loss:0.3294, validation_loss:0.3684, val_accuracy:0.8312\n",
      "Epoch [156/200], average_loss:0.3298, validation_loss:0.3622, val_accuracy:0.8831\n",
      "Epoch [157/200], average_loss:0.3296, validation_loss:0.3683, val_accuracy:0.8831\n",
      "Epoch [158/200], average_loss:0.3274, validation_loss:0.3655, val_accuracy:0.8961\n",
      "Epoch [159/200], average_loss:0.3284, validation_loss:0.3720, val_accuracy:0.8701\n",
      "Epoch [160/200], average_loss:0.3294, validation_loss:0.3738, val_accuracy:0.8052\n",
      "Epoch [161/200], average_loss:0.3301, validation_loss:0.3694, val_accuracy:0.8182\n",
      "Epoch [162/200], average_loss:0.3294, validation_loss:0.3656, val_accuracy:0.8442\n",
      "Epoch [163/200], average_loss:0.3281, validation_loss:0.3710, val_accuracy:0.7662\n",
      "Epoch [164/200], average_loss:0.3285, validation_loss:0.3681, val_accuracy:0.8312\n",
      "Epoch [165/200], average_loss:0.3260, validation_loss:0.3646, val_accuracy:0.8831\n",
      "Epoch [166/200], average_loss:0.3287, validation_loss:0.3707, val_accuracy:0.8182\n",
      "Epoch [167/200], average_loss:0.3294, validation_loss:0.3623, val_accuracy:0.8701\n",
      "Epoch [168/200], average_loss:0.3271, validation_loss:0.3640, val_accuracy:0.8831\n",
      "Epoch [169/200], average_loss:0.3267, validation_loss:0.3660, val_accuracy:0.8831\n",
      "Epoch [170/200], average_loss:0.3259, validation_loss:0.3647, val_accuracy:0.8701\n",
      "Epoch [171/200], average_loss:0.3296, validation_loss:0.3640, val_accuracy:0.9091\n",
      "Epoch [172/200], average_loss:0.3282, validation_loss:0.3680, val_accuracy:0.8182\n",
      "Epoch [173/200], average_loss:0.3258, validation_loss:0.3693, val_accuracy:0.8182\n",
      "Epoch [174/200], average_loss:0.3260, validation_loss:0.3803, val_accuracy:0.8442\n",
      "Epoch [175/200], average_loss:0.3277, validation_loss:0.3709, val_accuracy:0.8312\n",
      "Epoch [176/200], average_loss:0.3255, validation_loss:0.3682, val_accuracy:0.8442\n",
      "Epoch [177/200], average_loss:0.3263, validation_loss:0.3635, val_accuracy:0.8961\n",
      "Epoch [178/200], average_loss:0.3265, validation_loss:0.3701, val_accuracy:0.7922\n",
      "Epoch [179/200], average_loss:0.3268, validation_loss:0.3675, val_accuracy:0.8701\n",
      "Epoch [180/200], average_loss:0.3258, validation_loss:0.3728, val_accuracy:0.7662\n",
      "Epoch [181/200], average_loss:0.3262, validation_loss:0.3665, val_accuracy:0.8052\n",
      "Epoch [182/200], average_loss:0.3261, validation_loss:0.3684, val_accuracy:0.8571\n",
      "Epoch [183/200], average_loss:0.3255, validation_loss:0.3668, val_accuracy:0.8571\n",
      "Epoch [184/200], average_loss:0.3274, validation_loss:0.3681, val_accuracy:0.8442\n",
      "Epoch [185/200], average_loss:0.3252, validation_loss:0.3653, val_accuracy:0.8831\n",
      "Epoch [186/200], average_loss:0.3260, validation_loss:0.3694, val_accuracy:0.8701\n",
      "Epoch [187/200], average_loss:0.3266, validation_loss:0.3636, val_accuracy:0.8831\n",
      "Epoch [188/200], average_loss:0.3253, validation_loss:0.3713, val_accuracy:0.8052\n",
      "Epoch [189/200], average_loss:0.3240, validation_loss:0.3682, val_accuracy:0.8442\n",
      "Epoch [190/200], average_loss:0.3257, validation_loss:0.3664, val_accuracy:0.8831\n",
      "Epoch [191/200], average_loss:0.3234, validation_loss:0.3659, val_accuracy:0.8571\n",
      "Epoch [192/200], average_loss:0.3270, validation_loss:0.3672, val_accuracy:0.8571\n",
      "Epoch [193/200], average_loss:0.3253, validation_loss:0.3717, val_accuracy:0.7792\n",
      "Epoch [194/200], average_loss:0.3262, validation_loss:0.3680, val_accuracy:0.8701\n",
      "Epoch [195/200], average_loss:0.3267, validation_loss:0.3685, val_accuracy:0.8442\n",
      "Epoch [196/200], average_loss:0.3254, validation_loss:0.3687, val_accuracy:0.8442\n",
      "Epoch [197/200], average_loss:0.3266, validation_loss:0.3685, val_accuracy:0.8052\n",
      "Epoch [198/200], average_loss:0.3264, validation_loss:0.3696, val_accuracy:0.8442\n",
      "Epoch [199/200], average_loss:0.3234, validation_loss:0.3618, val_accuracy:0.8961\n",
      "Epoch [200/200], average_loss:0.3245, validation_loss:0.3723, val_accuracy:0.7792\n"
     ]
    }
   ],
   "source": [
    "lh = m1(epochs=range(100,200), train_loader=train_loader, validation_loader=valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dl-v7vuW7ZGJ"
   },
   "outputs": [],
   "source": [
    "m1.tb.close()\n",
    "\n",
    "m1.model.eval()\n",
    "with torch.no_grad():\n",
    "    x_pred = np.argmax(m1.model.clf(torch.tensor(X_test).float().to(m1.device)).cpu().detach(), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "nVJcVBba7ZQj",
    "outputId": "ad9c9adf-e291-4a61-a997-f61280318e43"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.832695153787287"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred=x_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "UpfUu-pxOFiW",
    "outputId": "93e7f8c2-3088-4f86-fd14-ba7e644eb74d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8608389101338432"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "TBNwia05OFl4",
    "outputId": "bcfd8546-69ad-44f6-8f01-f3451a830d2f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3076, 1030,   21,  108],\n",
       "       [ 625, 9693,   29,   98],\n",
       "       [  21,   37,  400,   21],\n",
       "       [ 195,  136,    8, 1238]], dtype=int64)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.save(m1.model.clf, \"models_and_losses/CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300+DNN_on_CNN-VAE_sampling_latent_space_m.IV_lr-{1e-2,1e-3}_bs-512_epoch-200.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# bs 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset = train, batch_size=512, shuffle=True)\n",
    "valid_loader = torch.utils.data.DataLoader(dataset = valid, batch_size=512, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f0r4jLHe5tPy"
   },
   "outputs": [],
   "source": [
    "DNN2 = nn.Sequential(\n",
    "            nn.Linear(in_features=15, out_features=64),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Dropout(p=0.3),\n",
    "            nn.Linear(in_features=64, out_features=128),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Dropout(p=0.3),\n",
    "            nn.Linear(in_features=128, out_features=4),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "-sou9uan7bFD",
    "outputId": "473dc211-21a7-4a1c-f9b7-1a9418fcc45e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder frozen.\n"
     ]
    }
   ],
   "source": [
    "model2 = Encoder_Classifier(encoder=VAE.encoder, clf = DNN2)\n",
    "model2.freeze_encoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xw7AbVFD7TKP"
   },
   "outputs": [],
   "source": [
    "optimizer= torch.optim.Adam(model2.parameters(), lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "hfD_WT5J7Yb-",
    "outputId": "0803e7a8-0aab-455e-9878-315ed7f840ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "m2 = Trainer(\n",
    "        model=model2,\n",
    "        optimizer=optimizer,\n",
    "        loss_function=nn.CrossEntropyLoss(),\n",
    "        scheduler=torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30], gamma=0.1),\n",
    "        tensorboard=True,\n",
    "        model_name=\"DNN_on_CNN-VAE_sampling_latent_space_m.VI_lr-{1e-2,1e-3}_bs-512\",\n",
    "        verbose=True\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n",
      "\t%eps : Float(512, 15) = aten::randn_like(%sigma, %135, %136, %137, %138), scope: Encoder_Classifier/Sequential[encoder]/VariationalLayer[12] # Z:\\Semi-supervised-Learning-with-VAE\\utils\\layers.py:35:0\n",
      "This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace()\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[333, 0] (0.668664276599884 vs. 0.7393266558647156) and 2040 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[369, 0] (0.7346095442771912 vs. 0.8613201975822449) and 2044 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[453, 0] (0.7637168169021606 vs. 1.0374982357025146) and 2045 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[204, 0] (0.7144594192504883 vs. 0.19462727010250092) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[69, 1] (1.3792154788970947 vs. 0.8213768005371094) and 2044 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[235, 2] (-0.9347221255302429 vs. -1.5176680088043213) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[224, 3] (0.914105236530304 vs. -0.08447495102882385) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[53, 3] (-0.3719034492969513 vs. 0.7363753914833069) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[200, 3] (-1.9270247220993042 vs. -0.22423455119132996) and 2047 other locations (100.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n",
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\jit\\__init__.py:1007: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n",
      "Not within tolerance rtol=1e-05 atol=1e-05 at input[185, 2] (-0.6067274212837219 vs. -2.4109580516815186) and 2046 other locations (99.00%)\n",
      "  check_tolerance, _force_outplace, True, _module_class)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], average_loss:0.8229, validation_loss:0.6216, val_accuracy:0.7532\n",
      "Epoch [2/100], average_loss:0.6354, validation_loss:0.5787, val_accuracy:0.7922\n",
      "Epoch [3/100], average_loss:0.5980, validation_loss:0.5454, val_accuracy:0.8182\n",
      "Epoch [4/100], average_loss:0.5712, validation_loss:0.5322, val_accuracy:0.8052\n",
      "Epoch [5/100], average_loss:0.5541, validation_loss:0.5124, val_accuracy:0.8442\n",
      "Epoch [6/100], average_loss:0.5435, validation_loss:0.5006, val_accuracy:0.8052\n",
      "Epoch [7/100], average_loss:0.5318, validation_loss:0.4898, val_accuracy:0.8312\n",
      "Epoch [8/100], average_loss:0.5194, validation_loss:0.4809, val_accuracy:0.8442\n",
      "Epoch [9/100], average_loss:0.5071, validation_loss:0.4609, val_accuracy:0.7922\n",
      "Epoch [10/100], average_loss:0.4951, validation_loss:0.4464, val_accuracy:0.8442\n",
      "Epoch [11/100], average_loss:0.4900, validation_loss:0.4400, val_accuracy:0.7662\n",
      "Epoch [12/100], average_loss:0.4818, validation_loss:0.4383, val_accuracy:0.8052\n",
      "Epoch [13/100], average_loss:0.4765, validation_loss:0.4296, val_accuracy:0.8182\n",
      "Epoch [14/100], average_loss:0.4703, validation_loss:0.4223, val_accuracy:0.8442\n",
      "Epoch [15/100], average_loss:0.4693, validation_loss:0.4317, val_accuracy:0.8312\n",
      "Epoch [16/100], average_loss:0.4688, validation_loss:0.4290, val_accuracy:0.8182\n",
      "Epoch [17/100], average_loss:0.4637, validation_loss:0.4322, val_accuracy:0.7532\n",
      "Epoch [18/100], average_loss:0.4617, validation_loss:0.4200, val_accuracy:0.8182\n",
      "Epoch [19/100], average_loss:0.4604, validation_loss:0.4192, val_accuracy:0.7922\n",
      "Epoch [20/100], average_loss:0.4569, validation_loss:0.4221, val_accuracy:0.9091\n",
      "Epoch [21/100], average_loss:0.4551, validation_loss:0.4172, val_accuracy:0.8961\n",
      "Epoch [22/100], average_loss:0.4559, validation_loss:0.4164, val_accuracy:0.8442\n",
      "Epoch [23/100], average_loss:0.4529, validation_loss:0.4171, val_accuracy:0.8701\n",
      "Epoch [24/100], average_loss:0.4504, validation_loss:0.4118, val_accuracy:0.8052\n",
      "Epoch [25/100], average_loss:0.4498, validation_loss:0.4141, val_accuracy:0.8052\n",
      "Epoch [26/100], average_loss:0.4476, validation_loss:0.4100, val_accuracy:0.8571\n",
      "Epoch [27/100], average_loss:0.4491, validation_loss:0.4122, val_accuracy:0.8312\n",
      "Epoch [28/100], average_loss:0.4469, validation_loss:0.4112, val_accuracy:0.8182\n",
      "Epoch [29/100], average_loss:0.4469, validation_loss:0.4064, val_accuracy:0.8831\n",
      "Epoch [30/100], average_loss:0.4437, validation_loss:0.4108, val_accuracy:0.8182\n",
      "Epoch [31/100], average_loss:0.4389, validation_loss:0.4050, val_accuracy:0.8182\n",
      "Epoch [32/100], average_loss:0.4379, validation_loss:0.4091, val_accuracy:0.7662\n",
      "Epoch [33/100], average_loss:0.4363, validation_loss:0.4018, val_accuracy:0.8831\n",
      "Epoch [34/100], average_loss:0.4365, validation_loss:0.4044, val_accuracy:0.8961\n",
      "Epoch [35/100], average_loss:0.4371, validation_loss:0.4022, val_accuracy:0.8442\n",
      "Epoch [36/100], average_loss:0.4356, validation_loss:0.4050, val_accuracy:0.7662\n",
      "Epoch [37/100], average_loss:0.4349, validation_loss:0.4021, val_accuracy:0.8052\n",
      "Epoch [38/100], average_loss:0.4345, validation_loss:0.4045, val_accuracy:0.8052\n",
      "Epoch [39/100], average_loss:0.4365, validation_loss:0.4034, val_accuracy:0.8052\n",
      "Epoch [40/100], average_loss:0.4328, validation_loss:0.4040, val_accuracy:0.7922\n",
      "Epoch [41/100], average_loss:0.4351, validation_loss:0.4031, val_accuracy:0.8182\n",
      "Epoch [42/100], average_loss:0.4333, validation_loss:0.4031, val_accuracy:0.8571\n",
      "Epoch [43/100], average_loss:0.4352, validation_loss:0.4042, val_accuracy:0.8442\n",
      "Epoch [44/100], average_loss:0.4367, validation_loss:0.4022, val_accuracy:0.8182\n",
      "Epoch [45/100], average_loss:0.4326, validation_loss:0.4052, val_accuracy:0.7273\n",
      "Epoch [46/100], average_loss:0.4372, validation_loss:0.4078, val_accuracy:0.7273\n",
      "Epoch [47/100], average_loss:0.4336, validation_loss:0.4035, val_accuracy:0.8182\n",
      "Epoch [48/100], average_loss:0.4347, validation_loss:0.4045, val_accuracy:0.8571\n",
      "Epoch [49/100], average_loss:0.4363, validation_loss:0.4048, val_accuracy:0.8182\n",
      "Epoch [50/100], average_loss:0.4338, validation_loss:0.4028, val_accuracy:0.8312\n",
      "Epoch [51/100], average_loss:0.4334, validation_loss:0.3992, val_accuracy:0.9221\n",
      "Epoch [52/100], average_loss:0.4347, validation_loss:0.3993, val_accuracy:0.8961\n",
      "Epoch [53/100], average_loss:0.4334, validation_loss:0.4029, val_accuracy:0.8442\n",
      "Epoch [54/100], average_loss:0.4348, validation_loss:0.3965, val_accuracy:0.9091\n",
      "Epoch [55/100], average_loss:0.4346, validation_loss:0.4035, val_accuracy:0.7662\n",
      "Epoch [56/100], average_loss:0.4349, validation_loss:0.4003, val_accuracy:0.8701\n",
      "Epoch [57/100], average_loss:0.4318, validation_loss:0.4026, val_accuracy:0.8312\n",
      "Epoch [58/100], average_loss:0.4341, validation_loss:0.4063, val_accuracy:0.8442\n",
      "Epoch [59/100], average_loss:0.4338, validation_loss:0.3972, val_accuracy:0.9091\n",
      "Epoch [60/100], average_loss:0.4318, validation_loss:0.4011, val_accuracy:0.8442\n",
      "Epoch [61/100], average_loss:0.4340, validation_loss:0.4010, val_accuracy:0.8182\n",
      "Epoch [62/100], average_loss:0.4316, validation_loss:0.4020, val_accuracy:0.8442\n",
      "Epoch [63/100], average_loss:0.4310, validation_loss:0.4020, val_accuracy:0.8182\n",
      "Epoch [64/100], average_loss:0.4327, validation_loss:0.4017, val_accuracy:0.7922\n",
      "Epoch [65/100], average_loss:0.4332, validation_loss:0.4025, val_accuracy:0.8052\n",
      "Epoch [66/100], average_loss:0.4318, validation_loss:0.3988, val_accuracy:0.8701\n",
      "Epoch [67/100], average_loss:0.4308, validation_loss:0.4015, val_accuracy:0.7922\n",
      "Epoch [68/100], average_loss:0.4297, validation_loss:0.4033, val_accuracy:0.8312\n",
      "Epoch [69/100], average_loss:0.4324, validation_loss:0.4002, val_accuracy:0.8312\n",
      "Epoch [70/100], average_loss:0.4297, validation_loss:0.3964, val_accuracy:0.8831\n",
      "Epoch [71/100], average_loss:0.4302, validation_loss:0.3977, val_accuracy:0.8701\n",
      "Epoch [72/100], average_loss:0.4318, validation_loss:0.4017, val_accuracy:0.8052\n",
      "Epoch [73/100], average_loss:0.4316, validation_loss:0.4015, val_accuracy:0.8052\n",
      "Epoch [74/100], average_loss:0.4295, validation_loss:0.3971, val_accuracy:0.8701\n",
      "Epoch [75/100], average_loss:0.4316, validation_loss:0.4008, val_accuracy:0.8182\n",
      "Epoch [76/100], average_loss:0.4295, validation_loss:0.3960, val_accuracy:0.9221\n",
      "Epoch [77/100], average_loss:0.4320, validation_loss:0.4004, val_accuracy:0.8052\n",
      "Epoch [78/100], average_loss:0.4301, validation_loss:0.3984, val_accuracy:0.8571\n",
      "Epoch [79/100], average_loss:0.4311, validation_loss:0.3998, val_accuracy:0.8312\n",
      "Epoch [80/100], average_loss:0.4284, validation_loss:0.3985, val_accuracy:0.8961\n",
      "Epoch [81/100], average_loss:0.4288, validation_loss:0.3994, val_accuracy:0.8701\n",
      "Epoch [82/100], average_loss:0.4316, validation_loss:0.4006, val_accuracy:0.8182\n",
      "Epoch [83/100], average_loss:0.4308, validation_loss:0.4011, val_accuracy:0.8571\n",
      "Epoch [84/100], average_loss:0.4320, validation_loss:0.4009, val_accuracy:0.8701\n",
      "Epoch [85/100], average_loss:0.4302, validation_loss:0.3977, val_accuracy:0.8571\n",
      "Epoch [86/100], average_loss:0.4309, validation_loss:0.3983, val_accuracy:0.8312\n",
      "Epoch [87/100], average_loss:0.4287, validation_loss:0.3987, val_accuracy:0.8182\n",
      "Epoch [88/100], average_loss:0.4310, validation_loss:0.4004, val_accuracy:0.8701\n",
      "Epoch [89/100], average_loss:0.4293, validation_loss:0.3985, val_accuracy:0.8182\n",
      "Epoch [90/100], average_loss:0.4288, validation_loss:0.3987, val_accuracy:0.8571\n",
      "Epoch [91/100], average_loss:0.4308, validation_loss:0.4025, val_accuracy:0.8312\n",
      "Epoch [92/100], average_loss:0.4271, validation_loss:0.4009, val_accuracy:0.8312\n",
      "Epoch [93/100], average_loss:0.4285, validation_loss:0.4018, val_accuracy:0.7662\n",
      "Epoch [94/100], average_loss:0.4276, validation_loss:0.3976, val_accuracy:0.8831\n",
      "Epoch [95/100], average_loss:0.4290, validation_loss:0.4031, val_accuracy:0.7922\n",
      "Epoch [96/100], average_loss:0.4303, validation_loss:0.3990, val_accuracy:0.8312\n",
      "Epoch [97/100], average_loss:0.4292, validation_loss:0.3991, val_accuracy:0.8961\n",
      "Epoch [98/100], average_loss:0.4288, validation_loss:0.3990, val_accuracy:0.8052\n",
      "Epoch [99/100], average_loss:0.4282, validation_loss:0.3951, val_accuracy:0.8571\n",
      "Epoch [100/100], average_loss:0.4295, validation_loss:0.3979, val_accuracy:0.8182\n"
     ]
    }
   ],
   "source": [
    "lh = m2(epochs=100, train_loader=train_loader, validation_loader=valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [101/200], average_loss:0.4283, validation_loss:0.4016, val_accuracy:0.8052\n",
      "Epoch [102/200], average_loss:0.4296, validation_loss:0.4009, val_accuracy:0.8182\n",
      "Epoch [103/200], average_loss:0.4294, validation_loss:0.3961, val_accuracy:0.8442\n",
      "Epoch [104/200], average_loss:0.4246, validation_loss:0.3975, val_accuracy:0.8052\n",
      "Epoch [105/200], average_loss:0.4273, validation_loss:0.3935, val_accuracy:0.9610\n",
      "Epoch [106/200], average_loss:0.4283, validation_loss:0.3989, val_accuracy:0.8442\n",
      "Epoch [107/200], average_loss:0.4278, validation_loss:0.4005, val_accuracy:0.7532\n",
      "Epoch [108/200], average_loss:0.4268, validation_loss:0.3997, val_accuracy:0.8182\n",
      "Epoch [109/200], average_loss:0.4275, validation_loss:0.3998, val_accuracy:0.8442\n",
      "Epoch [110/200], average_loss:0.4268, validation_loss:0.3942, val_accuracy:0.8701\n",
      "Epoch [111/200], average_loss:0.4292, validation_loss:0.3973, val_accuracy:0.8831\n",
      "Epoch [112/200], average_loss:0.4267, validation_loss:0.3969, val_accuracy:0.8701\n",
      "Epoch [113/200], average_loss:0.4267, validation_loss:0.3966, val_accuracy:0.8442\n",
      "Epoch [114/200], average_loss:0.4273, validation_loss:0.3960, val_accuracy:0.8961\n",
      "Epoch [115/200], average_loss:0.4265, validation_loss:0.4000, val_accuracy:0.7792\n",
      "Epoch [116/200], average_loss:0.4283, validation_loss:0.3953, val_accuracy:0.8312\n",
      "Epoch [117/200], average_loss:0.4277, validation_loss:0.3977, val_accuracy:0.8701\n",
      "Epoch [118/200], average_loss:0.4303, validation_loss:0.3969, val_accuracy:0.8442\n",
      "Epoch [119/200], average_loss:0.4257, validation_loss:0.4001, val_accuracy:0.7922\n",
      "Epoch [120/200], average_loss:0.4286, validation_loss:0.3962, val_accuracy:0.8442\n",
      "Epoch [121/200], average_loss:0.4300, validation_loss:0.3977, val_accuracy:0.8312\n",
      "Epoch [122/200], average_loss:0.4273, validation_loss:0.3988, val_accuracy:0.8571\n",
      "Epoch [123/200], average_loss:0.4282, validation_loss:0.3983, val_accuracy:0.8052\n",
      "Epoch [124/200], average_loss:0.4260, validation_loss:0.3993, val_accuracy:0.7922\n",
      "Epoch [125/200], average_loss:0.4251, validation_loss:0.3961, val_accuracy:0.8701\n",
      "Epoch [126/200], average_loss:0.4289, validation_loss:0.3973, val_accuracy:0.8701\n",
      "Epoch [127/200], average_loss:0.4267, validation_loss:0.3994, val_accuracy:0.8312\n",
      "Epoch [128/200], average_loss:0.4270, validation_loss:0.3972, val_accuracy:0.8182\n",
      "Epoch [129/200], average_loss:0.4298, validation_loss:0.4001, val_accuracy:0.8052\n",
      "Epoch [130/200], average_loss:0.4270, validation_loss:0.3996, val_accuracy:0.7792\n",
      "Epoch [131/200], average_loss:0.4251, validation_loss:0.3968, val_accuracy:0.8442\n",
      "Epoch [132/200], average_loss:0.4246, validation_loss:0.4037, val_accuracy:0.7273\n",
      "Epoch [133/200], average_loss:0.4276, validation_loss:0.3972, val_accuracy:0.8442\n",
      "Epoch [134/200], average_loss:0.4249, validation_loss:0.3960, val_accuracy:0.8571\n",
      "Epoch [135/200], average_loss:0.4265, validation_loss:0.3944, val_accuracy:0.8701\n",
      "Epoch [136/200], average_loss:0.4247, validation_loss:0.3959, val_accuracy:0.8831\n",
      "Epoch [137/200], average_loss:0.4240, validation_loss:0.4025, val_accuracy:0.7792\n",
      "Epoch [138/200], average_loss:0.4253, validation_loss:0.3954, val_accuracy:0.8442\n",
      "Epoch [139/200], average_loss:0.4289, validation_loss:0.4064, val_accuracy:0.7273\n",
      "Epoch [140/200], average_loss:0.4258, validation_loss:0.3965, val_accuracy:0.8312\n",
      "Epoch [141/200], average_loss:0.4268, validation_loss:0.3996, val_accuracy:0.7662\n",
      "Epoch [142/200], average_loss:0.4281, validation_loss:0.3961, val_accuracy:0.8182\n",
      "Epoch [143/200], average_loss:0.4246, validation_loss:0.3940, val_accuracy:0.8442\n",
      "Epoch [144/200], average_loss:0.4245, validation_loss:0.3939, val_accuracy:0.8571\n",
      "Epoch [145/200], average_loss:0.4266, validation_loss:0.3931, val_accuracy:0.9091\n",
      "Epoch [146/200], average_loss:0.4278, validation_loss:0.3946, val_accuracy:0.8442\n",
      "Epoch [147/200], average_loss:0.4246, validation_loss:0.3926, val_accuracy:0.8701\n",
      "Epoch [148/200], average_loss:0.4269, validation_loss:0.3944, val_accuracy:0.8571\n",
      "Epoch [149/200], average_loss:0.4260, validation_loss:0.3940, val_accuracy:0.8182\n",
      "Epoch [150/200], average_loss:0.4230, validation_loss:0.3966, val_accuracy:0.8052\n",
      "Epoch [151/200], average_loss:0.4240, validation_loss:0.3949, val_accuracy:0.8052\n",
      "Epoch [152/200], average_loss:0.4257, validation_loss:0.3957, val_accuracy:0.8442\n",
      "Epoch [153/200], average_loss:0.4205, validation_loss:0.3964, val_accuracy:0.8182\n",
      "Epoch [154/200], average_loss:0.4238, validation_loss:0.3955, val_accuracy:0.8701\n",
      "Epoch [155/200], average_loss:0.4258, validation_loss:0.3967, val_accuracy:0.8701\n",
      "Epoch [156/200], average_loss:0.4238, validation_loss:0.3995, val_accuracy:0.7792\n",
      "Epoch [157/200], average_loss:0.4236, validation_loss:0.3981, val_accuracy:0.8571\n",
      "Epoch [158/200], average_loss:0.4255, validation_loss:0.3952, val_accuracy:0.8182\n",
      "Epoch [159/200], average_loss:0.4248, validation_loss:0.3965, val_accuracy:0.7792\n",
      "Epoch [160/200], average_loss:0.4258, validation_loss:0.3950, val_accuracy:0.8831\n",
      "Epoch [161/200], average_loss:0.4266, validation_loss:0.3990, val_accuracy:0.7792\n",
      "Epoch [162/200], average_loss:0.4219, validation_loss:0.4016, val_accuracy:0.7273\n",
      "Epoch [163/200], average_loss:0.4257, validation_loss:0.3973, val_accuracy:0.8831\n",
      "Epoch [164/200], average_loss:0.4213, validation_loss:0.3925, val_accuracy:0.8961\n",
      "Epoch [165/200], average_loss:0.4237, validation_loss:0.3932, val_accuracy:0.9091\n",
      "Epoch [166/200], average_loss:0.4244, validation_loss:0.3962, val_accuracy:0.7662\n",
      "Epoch [167/200], average_loss:0.4258, validation_loss:0.3907, val_accuracy:0.8831\n",
      "Epoch [168/200], average_loss:0.4240, validation_loss:0.3945, val_accuracy:0.8571\n",
      "Epoch [169/200], average_loss:0.4238, validation_loss:0.3951, val_accuracy:0.8571\n",
      "Epoch [170/200], average_loss:0.4246, validation_loss:0.3953, val_accuracy:0.8442\n",
      "Epoch [171/200], average_loss:0.4246, validation_loss:0.3918, val_accuracy:0.8831\n",
      "Epoch [172/200], average_loss:0.4237, validation_loss:0.3945, val_accuracy:0.8442\n",
      "Epoch [173/200], average_loss:0.4249, validation_loss:0.3932, val_accuracy:0.8701\n",
      "Epoch [174/200], average_loss:0.4231, validation_loss:0.3949, val_accuracy:0.8442\n",
      "Epoch [175/200], average_loss:0.4242, validation_loss:0.3944, val_accuracy:0.8052\n",
      "Epoch [176/200], average_loss:0.4243, validation_loss:0.3947, val_accuracy:0.8571\n",
      "Epoch [177/200], average_loss:0.4248, validation_loss:0.3956, val_accuracy:0.9091\n",
      "Epoch [178/200], average_loss:0.4241, validation_loss:0.3914, val_accuracy:0.8831\n",
      "Epoch [179/200], average_loss:0.4220, validation_loss:0.3938, val_accuracy:0.8312\n",
      "Epoch [180/200], average_loss:0.4242, validation_loss:0.3937, val_accuracy:0.8571\n",
      "Epoch [181/200], average_loss:0.4223, validation_loss:0.3907, val_accuracy:0.8571\n",
      "Epoch [182/200], average_loss:0.4214, validation_loss:0.3926, val_accuracy:0.8442\n",
      "Epoch [183/200], average_loss:0.4239, validation_loss:0.3931, val_accuracy:0.8571\n",
      "Epoch [184/200], average_loss:0.4214, validation_loss:0.3931, val_accuracy:0.9091\n",
      "Epoch [185/200], average_loss:0.4205, validation_loss:0.3966, val_accuracy:0.7922\n",
      "Epoch [186/200], average_loss:0.4230, validation_loss:0.3962, val_accuracy:0.7143\n",
      "Epoch [187/200], average_loss:0.4233, validation_loss:0.3907, val_accuracy:0.9221\n",
      "Epoch [188/200], average_loss:0.4226, validation_loss:0.3943, val_accuracy:0.8052\n",
      "Epoch [189/200], average_loss:0.4237, validation_loss:0.3923, val_accuracy:0.9091\n",
      "Epoch [190/200], average_loss:0.4232, validation_loss:0.3925, val_accuracy:0.8831\n",
      "Epoch [191/200], average_loss:0.4226, validation_loss:0.3907, val_accuracy:0.8701\n",
      "Epoch [192/200], average_loss:0.4233, validation_loss:0.3902, val_accuracy:0.8571\n",
      "Epoch [193/200], average_loss:0.4223, validation_loss:0.3977, val_accuracy:0.7273\n",
      "Epoch [194/200], average_loss:0.4219, validation_loss:0.3953, val_accuracy:0.8182\n",
      "Epoch [195/200], average_loss:0.4233, validation_loss:0.3960, val_accuracy:0.7792\n",
      "Epoch [196/200], average_loss:0.4211, validation_loss:0.3945, val_accuracy:0.8182\n",
      "Epoch [197/200], average_loss:0.4221, validation_loss:0.3914, val_accuracy:0.8961\n",
      "Epoch [198/200], average_loss:0.4225, validation_loss:0.3946, val_accuracy:0.8701\n",
      "Epoch [199/200], average_loss:0.4222, validation_loss:0.3897, val_accuracy:0.9091\n",
      "Epoch [200/200], average_loss:0.4209, validation_loss:0.3940, val_accuracy:0.8442\n"
     ]
    }
   ],
   "source": [
    "lh = m2(epochs=range(100,200), train_loader=train_loader, validation_loader=valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dl-v7vuW7ZGJ"
   },
   "outputs": [],
   "source": [
    "#m2.tb.close()\n",
    "m2.model.eval()\n",
    "with torch.no_grad():\n",
    "    x_pred = np.argmax(m2.model.clf(torch.tensor(X_test).float().to(m2.device)).cpu().detach(), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "nVJcVBba7ZQj",
    "outputId": "ad9c9adf-e291-4a61-a997-f61280318e43"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8091057190570117"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true=y_test, y_pred=x_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "UpfUu-pxOFiW",
    "outputId": "93e7f8c2-3088-4f86-fd14-ba7e644eb74d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8456620458891013"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "TBNwia05OFl4",
    "outputId": "bcfd8546-69ad-44f6-8f01-f3451a830d2f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2665, 1432,   27,  111],\n",
       "       [ 412, 9921,   26,   86],\n",
       "       [  18,   45,  395,   21],\n",
       "       [ 212,  178,   15, 1172]], dtype=int64)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true=y_test, y_pred=x_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "m2.tb.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ghost_000\\Miniconda3\\envs\\Pytorch\\lib\\site-packages\\torch\\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Dropout. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "#torch.save(m2.model.clf, \"models_and_losses/CNN-VAE_ReLU_NLL_lr-{1e-3,1e-4}_bs-512_epoch-300+DNN_on_CNN-VAE_sampling_latent_space_m.VI_lr-{1e-2,1e-3}_bs-512_epoch-200.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Classification on CNN-VAE latent space with NN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}