2_2_Music_VAE run 2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2.2 Music VAE",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PratikStar/google-colab/blob/main/2_2_Music_VAE%20run%202.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHIAgyMQE-60"
      },
      "source": [
        "### Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xSpskSmkGT4C",
        "outputId": "c4578b4b-c62a-457a-dac0-3e75629b621a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',  force_remount=True)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zdlnHSb1FDsJ"
      },
      "source": [
        "### Install tensorflow v2.4.0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vr15BDmekCAy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "eb4807ce-541a-4827-c713-395ed60469dd"
      },
      "source": [
        "!pip uninstall --yes tensorflow\n",
        "!pip install tensorflow==2.4.0"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: tensorflow 2.6.0\n",
            "Uninstalling tensorflow-2.6.0:\n",
            "  Successfully uninstalled tensorflow-2.6.0\n",
            "Collecting tensorflow==2.4.0\n",
            "  Downloading tensorflow-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl (394.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 394.7 MB 14 kB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.37.0)\n",
            "Collecting gast==0.3.3\n",
            "  Downloading gast-0.3.3-py2.py3-none-any.whl (9.7 kB)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.7.4.3)\n",
            "Collecting h5py~=2.10.0\n",
            "  Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 79.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.17.3)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.6.3)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12.1)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.3.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.0)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0rc0\n",
            "  Downloading tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 68.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.2.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.2)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.12.0)\n",
            "Collecting grpcio~=1.32.0\n",
            "  Downloading grpcio-1.32.0-cp37-cp37m-manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 83.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.15.0)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.6.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.19.5)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.8.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (57.4.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.4.6)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (3.3.4)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.6.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.35.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.7.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (4.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (3.5.0)\n",
            "Installing collected packages: grpcio, tensorflow-estimator, h5py, gast, tensorflow\n",
            "  Attempting uninstall: grpcio\n",
            "    Found existing installation: grpcio 1.40.0\n",
            "    Uninstalling grpcio-1.40.0:\n",
            "      Successfully uninstalled grpcio-1.40.0\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.6.0\n",
            "    Uninstalling tensorflow-estimator-2.6.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.6.0\n",
            "  Attempting uninstall: h5py\n",
            "    Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.4.0\n",
            "    Uninstalling gast-0.4.0:\n",
            "      Successfully uninstalled gast-0.4.0\n",
            "Successfully installed gast-0.3.3 grpcio-1.32.0 h5py-2.10.0 tensorflow-2.4.0 tensorflow-estimator-2.4.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gast",
                  "grpc",
                  "h5py",
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eTlUZZpvz9Xt"
      },
      "source": [
        "## Autoencoder Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VlxNWPvvVn1L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2bc3125f-9421-42b6-997e-964f1263f209"
      },
      "source": [
        "\"#@title\"\n",
        "import os\n",
        "import pickle\n",
        "import csv\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Input, Conv2D, ReLU, BatchNormalization, \\\n",
        "    Flatten, Dense, Reshape, Conv2DTranspose, Activation, Lambda\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.losses import MeanSquaredError\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "print(tf.__version__)\n",
        "tf.compat.v1.disable_eager_execution()\n",
        "\n",
        "\n",
        "class VAE:\n",
        "    \"\"\"\n",
        "    VAE represents a Deep Convolutional variational autoencoder architecture\n",
        "    with mirrored encoder and decoder components.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 input_shape,\n",
        "                 conv_filters,\n",
        "                 conv_kernels,\n",
        "                 conv_strides,\n",
        "                 latent_space_dim):\n",
        "        self.input_shape = input_shape\n",
        "        self.conv_filters = conv_filters\n",
        "        self.conv_kernels = conv_kernels \n",
        "        self.conv_strides = conv_strides \n",
        "        self.latent_space_dim = latent_space_dim \n",
        "        self.reconstruction_loss_weight = 1000000\n",
        "\n",
        "        self.encoder = None\n",
        "        self.decoder = None\n",
        "        self.model = None\n",
        "\n",
        "        self._num_conv_layers = len(conv_filters)\n",
        "        self._shape_before_bottleneck = None\n",
        "        self._model_input = None\n",
        "\n",
        "        self._build()\n",
        "    def get_config(self):\n",
        "        return {}\n",
        "    def summary(self):\n",
        "        self.encoder.summary()\n",
        "        self.decoder.summary()\n",
        "        self.model.summary()\n",
        "\n",
        "    def compile(self, learning_rate=0.0001):\n",
        "        optimizer = Adam(learning_rate=learning_rate)\n",
        "        self.model.compile(optimizer=optimizer,\n",
        "                           loss=self._calculate_combined_loss,\n",
        "                           metrics=[self._calculate_reconstruction_loss,\n",
        "                                    self._calculate_kl_loss])\n",
        "\n",
        "    def train(self, generator, dataset_length, batch_size, num_epochs, callbacks):\n",
        "        if dataset_length % batch_size != 0:\n",
        "            raise Exception(\"dataset_length % batch_size = \" + str(dataset_length % batch_size) + \" != 0. Exiting!!\")\n",
        "        return self.model.fit(x=generator,\n",
        "                       steps_per_epoch= dataset_length/batch_size,\n",
        "                       epochs=num_epochs,\n",
        "                       callbacks=callbacks\n",
        "              )\n",
        "\n",
        "    def save(self, save_folder=\".\"):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        self.model.save(save_folder)\n",
        "\n",
        "    def savehdf5(self, save_folder=\".\"):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        self.save_parameters(save_folder)\n",
        "        self._save_weights(save_folder)\n",
        "\n",
        "    def save_parameters(self, save_folder):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        parameters = [\n",
        "            self.input_shape,\n",
        "            self.conv_filters,\n",
        "            self.conv_kernels,\n",
        "            self.conv_strides,\n",
        "            self.latent_space_dim\n",
        "        ]\n",
        "        save_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        with open(save_path, \"wb\") as f:\n",
        "            print(parameters)\n",
        "            pickle.dump(parameters, f)\n",
        "\n",
        "    def _save_weights(self, save_folder):\n",
        "        save_path = os.path.join(save_folder, \"weights.h5\")\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "    def reconstruct(self, images):\n",
        "        latent_representations = self.encoder.predict(images)\n",
        "        reconstructed_images = self.decoder.predict(latent_representations)\n",
        "        return reconstructed_images, latent_representations\n",
        "\n",
        "    @classmethod\n",
        "    def load(cls, save_folder=\".\", weights_file_name=\"weights.h5\"):\n",
        "        parameters_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        weights_path = os.path.join(save_folder, weights_file_name)\n",
        "        if not os.path.exists(parameters_path) or not os.path.exists(weights_path):\n",
        "            return None\n",
        "        with open(parameters_path, \"rb\") as f:\n",
        "            parameters = pickle.load(f)\n",
        "        autoencoder = VAE(*parameters)\n",
        "        autoencoder.load_weights(weights_path)\n",
        "        return autoencoder\n",
        "\n",
        "    def load_weights(self, weights_path):\n",
        "        self.model.load_weights(weights_path)\n",
        "\n",
        "    def _calculate_combined_loss(self, y_target, y_predicted):\n",
        "        reconstruction_loss = self._calculate_reconstruction_loss(y_target, y_predicted)\n",
        "        kl_loss = self._calculate_kl_loss(y_target, y_predicted)\n",
        "        combined_loss = self.reconstruction_loss_weight * reconstruction_loss\\\n",
        "                                                         + kl_loss\n",
        "        return combined_loss\n",
        "\n",
        "    def _calculate_reconstruction_loss(self, y_target, y_predicted):\n",
        "        error = y_target - y_predicted\n",
        "        reconstruction_loss = K.mean(K.square(error), axis=[1, 2, 3])\n",
        "        return reconstruction_loss\n",
        "\n",
        "    def _calculate_kl_loss(self, y_target, y_predicted):\n",
        "        kl_loss = -0.5 * K.sum(1 + self.log_variance - K.square(self.mu) -\n",
        "                               K.exp(self.log_variance), axis=1)\n",
        "        return kl_loss\n",
        "\n",
        "    def _create_folder_if_it_doesnt_exist(self, folder):\n",
        "        if not os.path.exists(folder):\n",
        "            os.makedirs(folder)\n",
        "\n",
        "    def _build(self):\n",
        "        self._build_encoder()\n",
        "        self._build_decoder()\n",
        "        self._build_autoencoder()\n",
        "\n",
        "    def _build_autoencoder(self):\n",
        "        model_input = self._model_input\n",
        "        model_output = self.decoder(self.encoder(model_input))\n",
        "        self.model = Model(model_input, model_output, name=\"autoencoder\")\n",
        "\n",
        "    def _build_decoder(self):\n",
        "        decoder_input = self._add_decoder_input()\n",
        "        dense_layer = self._add_dense_layer(decoder_input)\n",
        "        reshape_layer = self._add_reshape_layer(dense_layer)\n",
        "        conv_transpose_layers = self._add_conv_transpose_layers(reshape_layer)\n",
        "        decoder_output = self._add_decoder_output(conv_transpose_layers)\n",
        "        self.decoder = Model(decoder_input, decoder_output, name=\"decoder\")\n",
        "\n",
        "    def _add_decoder_input(self):\n",
        "        return Input(shape=self.latent_space_dim, name=\"decoder_input\")\n",
        "\n",
        "    def _add_dense_layer(self, decoder_input):\n",
        "        num_neurons = np.prod(self._shape_before_bottleneck) # [1, 2, 4] -> 8\n",
        "        dense_layer = Dense(num_neurons, name=\"decoder_dense\")(decoder_input)\n",
        "        return dense_layer\n",
        "\n",
        "    def _add_reshape_layer(self, dense_layer):\n",
        "        return Reshape(self._shape_before_bottleneck)(dense_layer)\n",
        "\n",
        "    def _add_conv_transpose_layers(self, x):\n",
        "        \"\"\"Add conv transpose blocks.\"\"\"\n",
        "        # loop through all the conv layers in reverse order and stop at the\n",
        "        # first layer\n",
        "        for layer_index in reversed(range(1, self._num_conv_layers)):\n",
        "            x = self._add_conv_transpose_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_transpose_layer(self, layer_index, x):\n",
        "        layer_num = self._num_conv_layers - layer_index\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{layer_num}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        x = ReLU(name=f\"decoder_relu_{layer_num}\")(x)\n",
        "        x = BatchNormalization(name=f\"decoder_bn_{layer_num}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_decoder_output(self, x):\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=1,\n",
        "            kernel_size=self.conv_kernels[0],\n",
        "            strides=self.conv_strides[0],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{self._num_conv_layers}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        output_layer = Activation(\"sigmoid\", name=\"sigmoid_layer\")(x)\n",
        "        return output_layer\n",
        "\n",
        "    def _build_encoder(self):\n",
        "        encoder_input = self._add_encoder_input()\n",
        "        conv_layers = self._add_conv_layers(encoder_input)\n",
        "        bottleneck = self._add_bottleneck(conv_layers)\n",
        "        self._model_input = encoder_input\n",
        "        self.encoder = Model(encoder_input, bottleneck, name=\"encoder\")\n",
        "\n",
        "    def _add_encoder_input(self):\n",
        "        return Input(shape=self.input_shape, name=\"encoder_input\")\n",
        "\n",
        "    def _add_conv_layers(self, encoder_input):\n",
        "        \"\"\"Create all convolutional blocks in encoder.\"\"\"\n",
        "        x = encoder_input\n",
        "        for layer_index in range(self._num_conv_layers):\n",
        "            x = self._add_conv_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_layer(self, layer_index, x):\n",
        "        \"\"\"Add a convolutional block to a graph of layers, consisting of\n",
        "        conv 2d + ReLU + batch normalization.\n",
        "        \"\"\"\n",
        "        layer_number = layer_index + 1\n",
        "        conv_layer = Conv2D(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"encoder_conv_layer_{layer_number}\"\n",
        "        )\n",
        "        x = conv_layer(x)\n",
        "        x = ReLU(name=f\"encoder_relu_{layer_number}\")(x)\n",
        "        x = BatchNormalization(name=f\"encoder_bn_{layer_number}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_bottleneck(self, x):\n",
        "        \"\"\"Flatten data and add bottleneck with Guassian sampling (Dense\n",
        "        layer).\n",
        "        \"\"\"\n",
        "        self._shape_before_bottleneck = K.int_shape(x)[1:]\n",
        "        x = Flatten()(x)\n",
        "        self.mu = Dense(self.latent_space_dim, name=\"mu\")(x)\n",
        "        self.log_variance = Dense(self.latent_space_dim,\n",
        "                                  name=\"log_variance\")(x)\n",
        "\n",
        "        def sample_point_from_normal_distribution(args):\n",
        "            mu, log_variance = args\n",
        "            epsilon = K.random_normal(shape=K.shape(self.mu), mean=0.,\n",
        "                                      stddev=1.)\n",
        "            sampled_point = mu + K.exp(log_variance / 2) * epsilon\n",
        "            return sampled_point\n",
        "\n",
        "        x = Lambda(sample_point_from_normal_distribution,\n",
        "                   name=\"encoder_output\")([self.mu, self.log_variance])\n",
        "        return x\n"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.4.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QIykrk9Dz54n"
      },
      "source": [
        "## Load Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_4XDb66wU0qt",
        "outputId": "b30102be-2236-4114-ab8c-bb2578dc04dc"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "AMP_IDS = [i for i in range(1, 81)] # 1-80\n",
        "CLIP_IDS = [i for i in range(1, 13)] # 1-12\n",
        "BATCH_SIZE = 27\n",
        "\n",
        "# Number of Reamped Clips (ampids): 80\n",
        "# Number of (Sub)clips each Reamped Clip(clipids): 12\n",
        "# Number of windows each Clip(windowid): different for clip:\n",
        "#     1: 13\n",
        "#     2: 12\n",
        "#     3: 13\n",
        "#     4: 11\n",
        "#     5: 11\n",
        "#     6: 14\n",
        "#     7: 11\n",
        "#     8: 14\n",
        "#     9: 11\n",
        "#     10: 08\n",
        "#     11: 09\n",
        "#     12: 15\n",
        "\n",
        "# Null Spectrograms/Windows. To be disregarded while training\n",
        "# 01-12\n",
        "# 04-10\n",
        "# 06-00\n",
        "# 07-10\n",
        "# 08-13\n",
        "# 11-08\n",
        "# 12-14\n",
        "\n",
        "def get_valid_files(spectrograms_path, ampids, clipids):\n",
        "\n",
        "    # Error checking\n",
        "    # ampid check: 1-80\n",
        "    invalidampids = [i for i in ampids if i>80 or i<1]\n",
        "    if len(invalidampids) > 0:\n",
        "        raise Exception(\"Invalid ampids: \" + str(invalidampids))\n",
        "    # clipid check: 1-12\n",
        "    invalidclipids = [i for i in clipids if i>12 or i<1]\n",
        "    if len(invalidclipids) > 0:\n",
        "        raise Exception(\"Invalid clipids: \" + str(invalidclipids))\n",
        "\n",
        "    clips_to_remove = [\"01-12\", \"04-10\", \"06-00\", \"07-10\", \"08-13\", \"11-08\", \"12-14\"]\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        ds = []\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            regex = \"^(?!00000.*$).*\"\n",
        "            if re.match(regex, filename) and (ampid in ampids and clipid in clipids) and (\"%02d\" % clipid + \"-\" + \"%02d\" % windowid) not in clips_to_remove:\n",
        "                ds.append(filename)\n",
        "\n",
        "        dslen = len(ds)\n",
        "    return sorted(ds)\n",
        "\n",
        "def music_ds_generator(spectrograms_path, filenames, batchsize):\n",
        "        i = 0\n",
        "        dslen = len(filenames)\n",
        "        while True:\n",
        "            x, y = [], []\n",
        "            xfiles, yfiles = [], []\n",
        "\n",
        "            st = i\n",
        "            end = (i + batchsize) % dslen\n",
        "            i = (end ) % dslen\n",
        "            \n",
        "            if end < st:\n",
        "                xfiles = filenames[st:dslen]\n",
        "                xfiles.extend(filenames[0:end])\n",
        "            else:\n",
        "                xfiles = filenames[st: end]\n",
        "        \n",
        "            for filename in xfiles:\n",
        "                fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "                ampid = int(fn_split[0])\n",
        "                clipid = int(fn_split[1])\n",
        "                windowid = int(fn_split[2])\n",
        "                di_filename = \"00000-\" + \"%02d\" % clipid + \"-\" + \"%02d\" % windowid + \" DI.wav.npy\"\n",
        "\n",
        "                yfiles.append(di_filename)\n",
        "\n",
        "                filepath = os.path.join(spectrograms_path, filename)\n",
        "                di_filepath = os.path.join(spectrograms_path, di_filename)\n",
        "                \n",
        "                spectrogram = np.load(filepath)\n",
        "                di_spectrogram = np.load(di_filepath)\n",
        "                \n",
        "                x.append(spectrogram[..., np.newaxis])\n",
        "                y.append(di_spectrogram[..., np.newaxis])\n",
        "            yield (np.array(x), np.array(y))#, xfiles, yfiles\n",
        "\n",
        "\n",
        "def load_all_music_ds(spectrograms_path, ampids, clipids):\n",
        "\n",
        "    # ampid check: 1-80\n",
        "    invalidampids = [i for i in ampids if i>80 or i<1]\n",
        "    if len(invalidampids) > 0:\n",
        "        raise Exception(\"Invalid ampids: \" + str(invalidampids))\n",
        "    # clipid check: 1-12\n",
        "    invalidclipids = [i for i in clipids if i>12 or i<1]\n",
        "    if len(invalidclipids) > 0:\n",
        "        raise Exception(\"Invalid clipids: \" + str(invalidclipids))\n",
        "\n",
        "    # Creating a list of DI file names\n",
        "    x, y = [], []\n",
        "    xfiles, yfiles = [], []\n",
        "    dis = []\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "        for filename in filenames:\n",
        "            regex = \"^00000.*\"\n",
        "            if re.match(regex, filename):\n",
        "                dis.append(filename)\n",
        "\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            if ampid in ampids and clipid in clipids:\n",
        "                # print(filename)\n",
        "                xfiles.append(filename)\n",
        "                di_regex = \"^00000-\" + \"%02d\" % clipid + \"-\" + \"%02d\" % windowid + \".*\"\n",
        "                r = re.compile(di_regex)\n",
        "                di_filename = list(filter(r.match, dis))[0]\n",
        "                yfiles.append(di_filename)\n",
        "                # print(\"DI is: \" + di_filename)\n",
        "                filepath = os.path.join(root, filename)\n",
        "                di_filepath = os.path.join(root, di_filename)\n",
        "                \n",
        "                spectrogram = np.load(filepath) # (n_bins, n_frames, 1) \n",
        "                di_spectrogram = np.load(di_filepath) # (n_bins, n_frames, 1) \n",
        "                \n",
        "                x.append(spectrogram[..., np.newaxis])\n",
        "                y.append(di_spectrogram[..., np.newaxis])\n",
        "    return np.array(x), np.array(y), xfiles, yfiles\n",
        "\n",
        "# x, y, xfiles, yfiles = load_all_music_ds(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "dsfilenames = get_valid_files(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "dslen = len(dsfilenames)\n",
        "print(\"Total dataset size: \" + str(dslen))\n",
        "\n",
        "musicds_gen = music_ds_generator(SPECTROGRAMS_PATH, dsfilenames, BATCH_SIZE)\n"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total dataset size: 10800\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoxtrbGxba-l"
      },
      "source": [
        "## Music AutoEncoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_GyWzngW2FB"
      },
      "source": [
        "### Instantiate Or Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FSivqKQZbeEp",
        "outputId": "40b72198-a69f-4d9e-e901-37a3994257de"
      },
      "source": [
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "LEARNING_RATE = 0.0001\n",
        "\n",
        "musicae = VAE.load(MODEL_PATH, 'weights.h5')\n",
        "\n",
        "if musicae == None:\n",
        "    print(\"Instantiating the model\")\n",
        "    LATENT_SPACE_DIM= 16\n",
        "    CONV_FILTERS = (128, 64, 32, 32)\n",
        "    CONV_KERNELS = (3, 3, 3, 3)\n",
        "    CONV_STRIDES = (1, 2, 2, 1)\n",
        "\n",
        "    musicae = VAE(\n",
        "        input_shape=(256, 64, 1),\n",
        "        conv_filters= CONV_FILTERS, \n",
        "        conv_kernels= CONV_KERNELS,\n",
        "        conv_strides= CONV_STRIDES,\n",
        "        latent_space_dim=LATENT_SPACE_DIM\n",
        "    )\n",
        "    musicae.save_parameters(MODEL_PATH)\n",
        "\n",
        "musicae.compile(LEARNING_RATE)\n",
        "musicae.summary()"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Instantiating the model\n",
            "[(256, 64, 1), (128, 64, 32, 32), (3, 3, 3, 3), (1, 2, 2, 1), 16]\n",
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "encoder_input (InputLayer)      [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_1 (Conv2D)   (None, 256, 64, 128) 1280        encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_1 (ReLU)           (None, 256, 64, 128) 0           encoder_conv_layer_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_1 (BatchNormalizatio (None, 256, 64, 128) 512         encoder_relu_1[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_2 (Conv2D)   (None, 128, 32, 64)  73792       encoder_bn_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_2 (ReLU)           (None, 128, 32, 64)  0           encoder_conv_layer_2[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_2 (BatchNormalizatio (None, 128, 32, 64)  256         encoder_relu_2[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_3 (Conv2D)   (None, 64, 16, 32)   18464       encoder_bn_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_3 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_3[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_3 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_3[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_4 (Conv2D)   (None, 64, 16, 32)   9248        encoder_bn_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_4 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_4[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_4 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_4[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "flatten_16 (Flatten)            (None, 32768)        0           encoder_bn_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 16)           524304      flatten_16[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "log_variance (Dense)            (None, 16)           524304      flatten_16[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 16)           0           mu[0][0]                         \n",
            "                                                                 log_variance[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 1,152,416\n",
            "Trainable params: 1,151,904\n",
            "Non-trainable params: 512\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "decoder_input (InputLayer)   [(None, 16)]              0         \n",
            "_________________________________________________________________\n",
            "decoder_dense (Dense)        (None, 32768)             557056    \n",
            "_________________________________________________________________\n",
            "reshape_16 (Reshape)         (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 64, 16, 32)        9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_1 (ReLU)        (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_1 (BatchNormaliza (None, 64, 16, 32)        128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 128, 32, 32)       9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_2 (ReLU)        (None, 128, 32, 32)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_2 (BatchNormaliza (None, 128, 32, 32)       128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 64)       18496     \n",
            "_________________________________________________________________\n",
            "decoder_relu_3 (ReLU)        (None, 256, 64, 64)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_3 (BatchNormaliza (None, 256, 64, 64)       256       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 1)        577       \n",
            "_________________________________________________________________\n",
            "sigmoid_layer (Activation)   (None, 256, 64, 1)        0         \n",
            "=================================================================\n",
            "Total params: 595,137\n",
            "Trainable params: 594,881\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "Model: \"autoencoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "encoder_input (InputLayer)   [(None, 256, 64, 1)]      0         \n",
            "_________________________________________________________________\n",
            "encoder (Functional)         (None, 16)                1152416   \n",
            "_________________________________________________________________\n",
            "decoder (Functional)         (None, 256, 64, 1)        595137    \n",
            "=================================================================\n",
            "Total params: 1,747,553\n",
            "Trainable params: 1,746,785\n",
            "Non-trainable params: 768\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Da9mV_T8vIz9"
      },
      "source": [
        "### Train Model (Iterative)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cL91qUHevKwM",
        "outputId": "68f5e22c-44dc-4c50-83eb-ae766a435776"
      },
      "source": [
        "import pytz\n",
        "from datetime import datetime\n",
        "\n",
        "EPOCHS = 500\n",
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "ts = pytz.timezone('Asia/Tokyo').localize(datetime.now())\n",
        "tsf = ts.strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
        "MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "CHECKPOINT_PATH = os.path.join(MODEL_PATH, \"checkpoints-\" + tsf)\n",
        "\n",
        "if not os.path.exists(CHECKPOINT_PATH):\n",
        "    os.makedirs(CHECKPOINT_PATH)\n",
        "\n",
        "class CustomCallback(tf.keras.callbacks.Callback):\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        save_checkpoint_path = os.path.join(CHECKPOINT_PATH, \"weights-{}.h5\".format(epoch))\n",
        "        save_path = os.path.join(MODEL_PATH, \"weights.h5\")\n",
        "        self.model.save_weights(save_checkpoint_path)\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "history = musicae.train(\n",
        "    generator=musicds_gen, \n",
        "    dataset_length=dslen,\n",
        "    batch_size=BATCH_SIZE, \n",
        "    num_epochs=EPOCHS,\n",
        "    callbacks=[CustomCallback()]\n",
        "    )\n",
        "losses = history.history['loss']\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1488.0521 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 105.1709\n",
            "Epoch 2/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1471.8303 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 104.0078\n",
            "Epoch 3/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1460.0866 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 102.9931\n",
            "Epoch 4/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1452.6283 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 102.0903\n",
            "Epoch 5/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1439.5183 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 100.9676\n",
            "Epoch 6/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1428.6368 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 99.6862\n",
            "Epoch 7/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1419.0359 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 98.6484\n",
            "Epoch 8/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1417.2363 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 97.5851\n",
            "Epoch 9/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1430.5724 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 96.7564\n",
            "Epoch 10/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1431.7539 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 96.0216\n",
            "Epoch 11/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1428.0530 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 95.3686\n",
            "Epoch 12/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1412.0596 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 94.8337\n",
            "Epoch 13/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1390.0092 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 93.8995\n",
            "Epoch 14/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1375.5983 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 92.9812\n",
            "Epoch 15/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1373.3645 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 92.1758\n",
            "Epoch 16/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1373.6244 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 91.3895\n",
            "Epoch 17/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1385.0234 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 91.0358\n",
            "Epoch 18/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1387.9506 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 90.1579\n",
            "Epoch 19/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1380.7675 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 89.6476\n",
            "Epoch 20/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1365.7130 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 89.2072\n",
            "Epoch 21/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1362.9558 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 88.5763\n",
            "Epoch 22/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1355.5412 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 88.1050\n",
            "Epoch 23/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1349.8088 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 87.5284\n",
            "Epoch 24/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1339.0983 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.9308\n",
            "Epoch 25/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1344.6514 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.6074\n",
            "Epoch 26/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1344.3605 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.0180\n",
            "Epoch 27/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1334.3300 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 85.8109\n",
            "Epoch 28/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1320.4975 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 85.3903\n",
            "Epoch 29/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1309.2926 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 84.9995\n",
            "Epoch 30/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1301.9367 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 84.3750\n",
            "Epoch 31/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1300.2152 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.8902\n",
            "Epoch 32/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1302.0499 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.5089\n",
            "Epoch 33/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1305.5993 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.1714\n",
            "Epoch 34/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1311.0932 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.7733\n",
            "Epoch 35/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1306.5561 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.4949\n",
            "Epoch 36/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1293.4113 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.0760\n",
            "Epoch 37/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1288.8853 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.7494\n",
            "Epoch 38/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1278.8468 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.6867\n",
            "Epoch 39/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1275.2610 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.3026\n",
            "Epoch 40/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1275.3055 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.8496\n",
            "Epoch 41/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1280.6593 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.4083\n",
            "Epoch 42/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1288.2669 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.2643\n",
            "Epoch 43/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1280.3800 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.0727\n",
            "Epoch 44/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1270.0822 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.7664\n",
            "Epoch 45/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1261.3919 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.5253\n",
            "Epoch 46/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1258.8966 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.2967\n",
            "Epoch 47/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1260.2027 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.9966\n",
            "Epoch 48/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1266.3112 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.8956\n",
            "Epoch 49/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1267.9126 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.6588\n",
            "Epoch 50/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1257.4329 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.4251\n",
            "Epoch 51/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1241.8440 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.1146\n",
            "Epoch 52/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1231.3955 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.9791\n",
            "Epoch 53/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1226.4299 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 77.7038\n",
            "Epoch 54/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1229.7183 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.3616\n",
            "Epoch 55/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1238.3022 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.2569\n",
            "Epoch 56/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1248.5267 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.1028\n",
            "Epoch 57/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1247.6673 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.0849\n",
            "Epoch 58/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1233.6400 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 76.9184\n",
            "Epoch 59/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1216.4198 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.7914\n",
            "Epoch 60/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.1639 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.5288\n",
            "Epoch 61/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1209.9137 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.4167\n",
            "Epoch 62/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1219.2587 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.3816\n",
            "Epoch 63/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1212.8499 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.1847\n",
            "Epoch 64/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1210.8503 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.9336\n",
            "Epoch 65/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1200.8726 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.8394\n",
            "Epoch 66/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1193.7851 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.6978\n",
            "Epoch 67/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1192.3605 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.7112\n",
            "Epoch 68/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1196.0551 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.4597\n",
            "Epoch 69/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1201.4818 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.3352\n",
            "Epoch 70/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.8934 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.1214\n",
            "Epoch 71/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.6724 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0025\n",
            "Epoch 72/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1204.0853 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0853\n",
            "Epoch 73/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1197.4586 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0374\n",
            "Epoch 74/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1186.8137 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7726\n",
            "Epoch 75/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1182.8644 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7755\n",
            "Epoch 76/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1181.5693 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7043\n",
            "Epoch 77/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1178.4952 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.6631\n",
            "Epoch 78/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1178.2954 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.6399\n",
            "Epoch 79/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1167.8240 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.5870\n",
            "Epoch 80/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1168.4252 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.5512\n",
            "Epoch 81/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1172.3835 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.2442\n",
            "Epoch 82/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1174.9812 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1641\n",
            "Epoch 83/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1181.8707 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1714\n",
            "Epoch 84/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1171.7414 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1152\n",
            "Epoch 85/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1159.0319 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.0263\n",
            "Epoch 86/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1151.6525 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.9948\n",
            "Epoch 87/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.6282 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.9032\n",
            "Epoch 88/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1155.2452 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8921\n",
            "Epoch 89/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1161.5215 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8183\n",
            "Epoch 90/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1159.3842 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.7223\n",
            "Epoch 91/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1158.2553 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.6046\n",
            "Epoch 92/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.9864 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5733\n",
            "Epoch 93/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1145.8543 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8737\n",
            "Epoch 94/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1143.7265 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.6962\n",
            "Epoch 95/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.7854 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5416\n",
            "Epoch 96/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1155.3675 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.4670\n",
            "Epoch 97/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1142.6632 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5251\n",
            "Epoch 98/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1133.2788 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.4273\n",
            "Epoch 99/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1130.4256 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3696\n",
            "Epoch 100/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1131.6547 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3510\n",
            "Epoch 101/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1139.1575 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3376\n",
            "Epoch 102/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1147.4840 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1318\n",
            "Epoch 103/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1144.2501 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1650\n",
            "Epoch 104/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1140.7766 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1767\n",
            "Epoch 105/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1131.7070 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1996\n",
            "Epoch 106/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1127.8386 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1650\n",
            "Epoch 107/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1125.0092 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.9652\n",
            "Epoch 108/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1125.7291 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.0634\n",
            "Epoch 109/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1133.9799 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.8936\n",
            "Epoch 110/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1143.0066 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.0318\n",
            "Epoch 111/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1123.3852 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.9301\n",
            "Epoch 112/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1115.4454 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9099\n",
            "Epoch 113/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1110.1434 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9452\n",
            "Epoch 114/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1111.9042 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9209\n",
            "Epoch 115/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1119.0179 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9271\n",
            "Epoch 116/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1117.6802 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7479\n",
            "Epoch 117/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1116.9587 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8887\n",
            "Epoch 118/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1120.6272 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8293\n",
            "Epoch 119/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1118.1231 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6738\n",
            "Epoch 120/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1114.5213 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8280\n",
            "Epoch 121/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1108.2669 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7024\n",
            "Epoch 122/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1104.6713 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8149\n",
            "Epoch 123/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1108.0492 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6966\n",
            "Epoch 124/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1107.6960 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7316\n",
            "Epoch 125/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1107.0105 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6401\n",
            "Epoch 126/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1100.8753 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6655\n",
            "Epoch 127/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.3843 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7391\n",
            "Epoch 128/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1103.0008 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7375\n",
            "Epoch 129/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1112.2566 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6087\n",
            "Epoch 130/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1102.7844 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6344\n",
            "Epoch 131/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.2275 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6223\n",
            "Epoch 132/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1096.2896 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6710\n",
            "Epoch 133/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1101.8963 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6256\n",
            "Epoch 134/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1099.8813 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7072\n",
            "Epoch 135/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1098.0477 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7555\n",
            "Epoch 136/500\n",
            "400/400 [==============================] - 42s 106ms/step - batch: 199.5000 - size: 27.0000 - loss: 1098.2068 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4796\n",
            "Epoch 137/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1094.1453 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4994\n",
            "Epoch 138/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1087.2736 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5387\n",
            "Epoch 139/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1082.7307 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6684\n",
            "Epoch 140/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1081.6546 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6293\n",
            "Epoch 141/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1090.7304 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5566\n",
            "Epoch 142/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.2799 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4733\n",
            "Epoch 143/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1099.0105 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4738\n",
            "Epoch 144/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1088.6216 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4746\n",
            "Epoch 145/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1080.3324 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5152\n",
            "Epoch 146/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.6009 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6030\n",
            "Epoch 147/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.2412 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5904\n",
            "Epoch 148/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1085.1565 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7111\n",
            "Epoch 149/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1085.2162 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4100\n",
            "Epoch 150/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1078.4290 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4310\n",
            "Epoch 151/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.7161 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5440\n",
            "Epoch 152/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.0870 - _calculate_reconstruction_loss: 9.9960e-04 - _calculate_kl_loss: 72.4860\n",
            "Epoch 153/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1074.8103 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4638\n",
            "Epoch 154/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1083.9813 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4656\n",
            "Epoch 155/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1082.2500 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5027\n",
            "Epoch 156/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.3388 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5145\n",
            "Epoch 157/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1066.4024 - _calculate_reconstruction_loss: 9.9380e-04 - _calculate_kl_loss: 72.6005\n",
            "Epoch 158/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1059.6866 - _calculate_reconstruction_loss: 9.8709e-04 - _calculate_kl_loss: 72.5965\n",
            "Epoch 159/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1063.7009 - _calculate_reconstruction_loss: 9.9106e-04 - _calculate_kl_loss: 72.6433\n",
            "Epoch 160/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.7674 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4786\n",
            "Epoch 161/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1080.9924 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6030\n",
            "Epoch 162/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1073.6301 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.3756\n",
            "Epoch 163/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1067.6502 - _calculate_reconstruction_loss: 9.9527e-04 - _calculate_kl_loss: 72.3826\n",
            "Epoch 164/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1063.3289 - _calculate_reconstruction_loss: 9.9073e-04 - _calculate_kl_loss: 72.6010\n",
            "Epoch 165/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1066.1802 - _calculate_reconstruction_loss: 9.9353e-04 - _calculate_kl_loss: 72.6450\n",
            "Epoch 166/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1071.5878 - _calculate_reconstruction_loss: 9.9901e-04 - _calculate_kl_loss: 72.5737\n",
            "Epoch 167/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1076.5916 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4959\n",
            "Epoch 168/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1074.2866 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.3564\n",
            "Epoch 169/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1070.5372 - _calculate_reconstruction_loss: 9.9806e-04 - _calculate_kl_loss: 72.4797\n",
            "Epoch 170/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1058.5215 - _calculate_reconstruction_loss: 9.8605e-04 - _calculate_kl_loss: 72.4693\n",
            "Epoch 171/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.0255 - _calculate_reconstruction_loss: 9.8259e-04 - _calculate_kl_loss: 72.4321\n",
            "Epoch 172/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1052.7973 - _calculate_reconstruction_loss: 9.8020e-04 - _calculate_kl_loss: 72.5987\n",
            "Epoch 173/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1059.7720 - _calculate_reconstruction_loss: 9.8722e-04 - _calculate_kl_loss: 72.5474\n",
            "Epoch 174/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1064.3050 - _calculate_reconstruction_loss: 9.9172e-04 - _calculate_kl_loss: 72.5822\n",
            "Epoch 175/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1056.4016 - _calculate_reconstruction_loss: 9.8384e-04 - _calculate_kl_loss: 72.5582\n",
            "Epoch 176/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1054.6076 - _calculate_reconstruction_loss: 9.8203e-04 - _calculate_kl_loss: 72.5800\n",
            "Epoch 177/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.7931 - _calculate_reconstruction_loss: 9.8527e-04 - _calculate_kl_loss: 72.5233\n",
            "Epoch 178/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.2408 - _calculate_reconstruction_loss: 9.8461e-04 - _calculate_kl_loss: 72.6321\n",
            "Epoch 179/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.7577 - _calculate_reconstruction_loss: 9.8521e-04 - _calculate_kl_loss: 72.5495\n",
            "Epoch 180/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.1216 - _calculate_reconstruction_loss: 9.8255e-04 - _calculate_kl_loss: 72.5707\n",
            "Epoch 181/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1051.5832 - _calculate_reconstruction_loss: 9.7901e-04 - _calculate_kl_loss: 72.5731\n",
            "Epoch 182/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1052.9203 - _calculate_reconstruction_loss: 9.8033e-04 - _calculate_kl_loss: 72.5908\n",
            "Epoch 183/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1054.0274 - _calculate_reconstruction_loss: 9.8140e-04 - _calculate_kl_loss: 72.6233\n",
            "Epoch 184/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.4713 - _calculate_reconstruction_loss: 9.8295e-04 - _calculate_kl_loss: 72.5255\n",
            "Epoch 185/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1047.2690 - _calculate_reconstruction_loss: 9.7468e-04 - _calculate_kl_loss: 72.5900\n",
            "Epoch 186/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1045.7203 - _calculate_reconstruction_loss: 9.7306e-04 - _calculate_kl_loss: 72.6642\n",
            "Epoch 187/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1045.1950 - _calculate_reconstruction_loss: 9.7247e-04 - _calculate_kl_loss: 72.7217\n",
            "Epoch 188/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1048.8732 - _calculate_reconstruction_loss: 9.7618e-04 - _calculate_kl_loss: 72.6971\n",
            "Epoch 189/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.7260 - _calculate_reconstruction_loss: 9.8309e-04 - _calculate_kl_loss: 72.6321\n",
            "Epoch 190/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.2774 - _calculate_reconstruction_loss: 9.8472e-04 - _calculate_kl_loss: 72.5547\n",
            "Epoch 191/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1046.7461 - _calculate_reconstruction_loss: 9.7412e-04 - _calculate_kl_loss: 72.6270\n",
            "Epoch 192/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1042.2733 - _calculate_reconstruction_loss: 9.6961e-04 - _calculate_kl_loss: 72.6677\n",
            "Epoch 193/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1042.8918 - _calculate_reconstruction_loss: 9.7020e-04 - _calculate_kl_loss: 72.6962\n",
            "Epoch 194/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1049.9453 - _calculate_reconstruction_loss: 9.7724e-04 - _calculate_kl_loss: 72.7072\n",
            "Epoch 195/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1058.4816 - _calculate_reconstruction_loss: 9.8597e-04 - _calculate_kl_loss: 72.5088\n",
            "Epoch 196/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1051.2597 - _calculate_reconstruction_loss: 9.7869e-04 - _calculate_kl_loss: 72.5646\n",
            "Epoch 197/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.0685 - _calculate_reconstruction_loss: 9.6844e-04 - _calculate_kl_loss: 72.6315\n",
            "Epoch 198/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1038.6845 - _calculate_reconstruction_loss: 9.6602e-04 - _calculate_kl_loss: 72.6693\n",
            "Epoch 199/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.6902 - _calculate_reconstruction_loss: 9.6902e-04 - _calculate_kl_loss: 72.6713\n",
            "Epoch 200/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1049.1990 - _calculate_reconstruction_loss: 9.7647e-04 - _calculate_kl_loss: 72.7317\n",
            "Epoch 201/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.7459 - _calculate_reconstruction_loss: 9.6909e-04 - _calculate_kl_loss: 72.6534\n",
            "Epoch 202/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.1162 - _calculate_reconstruction_loss: 9.6240e-04 - _calculate_kl_loss: 72.7180\n",
            "Epoch 203/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.4035 - _calculate_reconstruction_loss: 9.6274e-04 - _calculate_kl_loss: 72.6663\n",
            "Epoch 204/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1040.6795 - _calculate_reconstruction_loss: 9.6788e-04 - _calculate_kl_loss: 72.8036\n",
            "Epoch 205/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1044.1715 - _calculate_reconstruction_loss: 9.7145e-04 - _calculate_kl_loss: 72.7223\n",
            "Epoch 206/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.2187 - _calculate_reconstruction_loss: 9.6844e-04 - _calculate_kl_loss: 72.7808\n",
            "Epoch 207/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1037.8321 - _calculate_reconstruction_loss: 9.6509e-04 - _calculate_kl_loss: 72.7458\n",
            "Epoch 208/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.4090 - _calculate_reconstruction_loss: 9.6171e-04 - _calculate_kl_loss: 72.6986\n",
            "Epoch 209/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.6216 - _calculate_reconstruction_loss: 9.6187e-04 - _calculate_kl_loss: 72.7516\n",
            "Epoch 210/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1038.4962 - _calculate_reconstruction_loss: 9.6575e-04 - _calculate_kl_loss: 72.7486\n",
            "Epoch 211/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1040.2035 - _calculate_reconstruction_loss: 9.6747e-04 - _calculate_kl_loss: 72.7334\n",
            "Epoch 212/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1037.8712 - _calculate_reconstruction_loss: 9.6512e-04 - _calculate_kl_loss: 72.7474\n",
            "Epoch 213/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.4892 - _calculate_reconstruction_loss: 9.6282e-04 - _calculate_kl_loss: 72.6646\n",
            "Epoch 214/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.4518 - _calculate_reconstruction_loss: 9.6178e-04 - _calculate_kl_loss: 72.6771\n",
            "Epoch 215/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1032.5443 - _calculate_reconstruction_loss: 9.5975e-04 - _calculate_kl_loss: 72.7933\n",
            "Epoch 216/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1031.6201 - _calculate_reconstruction_loss: 9.5882e-04 - _calculate_kl_loss: 72.8026\n",
            "Epoch 217/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1033.0277 - _calculate_reconstruction_loss: 9.6026e-04 - _calculate_kl_loss: 72.7654\n",
            "Epoch 218/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1036.4967 - _calculate_reconstruction_loss: 9.6364e-04 - _calculate_kl_loss: 72.8588\n",
            "Epoch 219/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1036.6911 - _calculate_reconstruction_loss: 9.6388e-04 - _calculate_kl_loss: 72.8090\n",
            "Epoch 220/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1029.8988 - _calculate_reconstruction_loss: 9.5714e-04 - _calculate_kl_loss: 72.7545\n",
            "Epoch 221/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.8293 - _calculate_reconstruction_loss: 9.5100e-04 - _calculate_kl_loss: 72.8320\n",
            "Epoch 222/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.5061 - _calculate_reconstruction_loss: 9.4957e-04 - _calculate_kl_loss: 72.9389\n",
            "Epoch 223/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.0864 - _calculate_reconstruction_loss: 9.5525e-04 - _calculate_kl_loss: 72.8339\n",
            "Epoch 224/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1030.2691 - _calculate_reconstruction_loss: 9.5752e-04 - _calculate_kl_loss: 72.7461\n",
            "Epoch 225/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1029.1620 - _calculate_reconstruction_loss: 9.5639e-04 - _calculate_kl_loss: 72.7715\n",
            "Epoch 226/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.0935 - _calculate_reconstruction_loss: 9.5532e-04 - _calculate_kl_loss: 72.7696\n",
            "Epoch 227/500\n",
            "400/400 [==============================] - 42s 106ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.2350 - _calculate_reconstruction_loss: 9.5034e-04 - _calculate_kl_loss: 72.8966\n",
            "Epoch 228/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1018.8728 - _calculate_reconstruction_loss: 9.4591e-04 - _calculate_kl_loss: 72.9612\n",
            "Epoch 229/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1021.9653 - _calculate_reconstruction_loss: 9.4908e-04 - _calculate_kl_loss: 72.8825\n",
            "Epoch 230/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.4029 - _calculate_reconstruction_loss: 9.5557e-04 - _calculate_kl_loss: 72.8344\n",
            "Epoch 231/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1026.9867 - _calculate_reconstruction_loss: 9.5413e-04 - _calculate_kl_loss: 72.8609\n",
            "Epoch 232/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.7346 - _calculate_reconstruction_loss: 9.5080e-04 - _calculate_kl_loss: 72.9389\n",
            "Epoch 233/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.6545 - _calculate_reconstruction_loss: 9.4975e-04 - _calculate_kl_loss: 72.9053\n",
            "Epoch 234/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1021.4211 - _calculate_reconstruction_loss: 9.4848e-04 - _calculate_kl_loss: 72.9424\n",
            "Epoch 235/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1019.1005 - _calculate_reconstruction_loss: 9.4604e-04 - _calculate_kl_loss: 73.0579\n",
            "Epoch 236/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.8446 - _calculate_reconstruction_loss: 9.4288e-04 - _calculate_kl_loss: 72.9602\n",
            "Epoch 237/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1024.5346 - _calculate_reconstruction_loss: 9.5151e-04 - _calculate_kl_loss: 73.0274\n",
            "Epoch 238/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1026.9122 - _calculate_reconstruction_loss: 9.5394e-04 - _calculate_kl_loss: 72.9723\n",
            "Epoch 239/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.1575 - _calculate_reconstruction_loss: 9.5023e-04 - _calculate_kl_loss: 72.9301\n",
            "Epoch 240/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1017.1436 - _calculate_reconstruction_loss: 9.4419e-04 - _calculate_kl_loss: 72.9572\n",
            "Epoch 241/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.2269 - _calculate_reconstruction_loss: 9.4220e-04 - _calculate_kl_loss: 73.0235\n",
            "Epoch 242/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.1304 - _calculate_reconstruction_loss: 9.4219e-04 - _calculate_kl_loss: 72.9401\n",
            "Epoch 243/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.2239 - _calculate_reconstruction_loss: 9.5015e-04 - _calculate_kl_loss: 73.0722\n",
            "Epoch 244/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1025.1777 - _calculate_reconstruction_loss: 9.5224e-04 - _calculate_kl_loss: 72.9346\n",
            "Epoch 245/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1018.3121 - _calculate_reconstruction_loss: 9.4529e-04 - _calculate_kl_loss: 73.0216\n",
            "Epoch 246/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1011.3814 - _calculate_reconstruction_loss: 9.3835e-04 - _calculate_kl_loss: 73.0284\n",
            "Epoch 247/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1009.5924 - _calculate_reconstruction_loss: 9.3650e-04 - _calculate_kl_loss: 73.0952\n",
            "Epoch 248/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1012.3390 - _calculate_reconstruction_loss: 9.3921e-04 - _calculate_kl_loss: 73.1248\n",
            "Epoch 249/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1017.9151 - _calculate_reconstruction_loss: 9.4488e-04 - _calculate_kl_loss: 73.0400\n",
            "Epoch 250/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.8747 - _calculate_reconstruction_loss: 9.4994e-04 - _calculate_kl_loss: 72.9398\n",
            "Epoch 251/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1019.5176 - _calculate_reconstruction_loss: 9.4653e-04 - _calculate_kl_loss: 72.9890\n",
            "Epoch 252/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1010.5522 - _calculate_reconstruction_loss: 9.3748e-04 - _calculate_kl_loss: 73.0720\n",
            "Epoch 253/500\n",
            "297/400 [=====================>........] - ETA: 10s - batch: 148.0000 - size: 27.0000 - loss: 1004.9426 - _calculate_reconstruction_loss: 9.3179e-04 - _calculate_kl_loss: 73.1512"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lg7LJLPT81b6"
      },
      "source": [
        "### Save Meta"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wvvbXQ1V82wM",
        "outputId": "9cea5ced-be2f-44a0-c61c-f1a42d04beca"
      },
      "source": [
        "# Training History\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "import gspread\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "gc = gspread.authorize(GoogleCredentials.get_application_default())\n",
        "ws = gc.open_by_url('https://docs.google.com/spreadsheets/d/12Q6uF-Oy8-Cq6AYFalYJzOjzXCMJhR8HDU3wTmygDmY/edit#gid=1946289387').sheet1\n",
        "all = ws.get_all_records()\n",
        "last_ts = all[-1]['Timestamp']\n",
        "ws.resize(len(all)+1)\n",
        "if last_ts != ts.strftime(\"%Y/%m/%d %H:%M:%S\"):\n",
        "    ws.append_row([\n",
        "                  ts.strftime(\"%Y/%m/%d %H:%M:%S\"),\n",
        "                  str(AMP_IDS),\n",
        "                  str(CLIP_IDS),\n",
        "                  str(CONV_FILTERS),\n",
        "                  str(CONV_KERNELS),\n",
        "                  str(CONV_STRIDES),\n",
        "                  str(LATENT_SPACE_DIM),\n",
        "                  str(LEARNING_RATE),\n",
        "                  str(BATCH_SIZE),\n",
        "                  str(EPOCHS),\n",
        "                  str(int(min(losses))),\n",
        "                  str(losses)\n",
        "    ])\n",
        "else:\n",
        "    print(\"Already added to Training history!\")\n",
        "\n",
        "print(\"Metadata Saved!!\")\n",
        "\n"
      ],
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Metadata Saved!!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-a2zPm2-nQd2"
      },
      "source": [
        "### [Broken] Generate the embeddings (Independently executable)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x2IhMH3guEs_",
        "outputId": "e2240ee3-6113-4966-a17e-afb85be22bbc"
      },
      "source": [
        "# # https://github.com/musikalkemist/generating-sound-with-neural-networks/blob/49d7db32c43d1a04c596cbbb282a9521be1e7fc8/11%20Implementing%20VAE/code/analysis.py\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa\n",
        "import librosa.display\n",
        "\n",
        "# TODO before running this cell independently\n",
        "# 1. Update the MODEL_NAME\n",
        "# 2. Run the \"Load Data\" cell\n",
        "\n",
        "# UPDATE THE MODEL!!!\n",
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "MODEL_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/model\"\n",
        "\n",
        "# Comes from the Load Data cell\n",
        "dsfilenames = list(set(yfiles)) + xfiles\n",
        "\n",
        "# Not tested!!\n",
        "def plot_reconstructed_images(images, reconstructed_images):\n",
        "    num_images = len(images)\n",
        "    for i, (image, reconstructed_image) in enumerate(zip(images, reconstructed_images)):\n",
        "\n",
        "        fig, ax = plt.subplots()\n",
        "        image = image.squeeze()\n",
        "        img = librosa.display.specshow(image, y_axis='log', x_axis='time', ax=ax)\n",
        "        fig.colorbar(img, ax=ax, format=\"%+2.0f dB\")\n",
        "\n",
        "        fig, ax = plt.subplots()\n",
        "        reconstructed_image = reconstructed_image.squeeze()\n",
        "        recon_img = librosa.display.specshow(reconstructed_image, y_axis='log', x_axis='time', ax=ax)\n",
        "        fig.colorbar(recon_img, ax=ax, format=\"%+2.0f dB\")\n",
        "    plt.show()\n",
        "\n",
        "# Not tested!!\n",
        "def plot_images_encoded_in_latent_space(latent_representations, sample_labels):\n",
        "    plt.figure(figsize=(10, 10))\n",
        "    plt.scatter(latent_representations[:, 0],\n",
        "                latent_representations[:, 1],\n",
        "                cmap=\"rainbow\",\n",
        "                c=sample_labels,\n",
        "                alpha=0.5,\n",
        "                s=2)\n",
        "    plt.colorbar()\n",
        "    plt.show()\n",
        "\n",
        "def save_embeddings(musicae, dsfilenames, download_path):\n",
        "\n",
        "\n",
        "    dsspectrogram = []\n",
        "    for filename in dsfilenames:\n",
        "        filepath = os.path.join(SPECTROGRAMS_PATH, filename)\n",
        "        spectrogram = np.load(filepath)\n",
        "        dsspectrogram.append(spectrogram[..., np.newaxis])\n",
        "\n",
        "    dsspectrogram = np.array(dsspectrogram)\n",
        "    latent_representations = musicae.encoder.predict(dsspectrogram)\n",
        "\n",
        "    with open(os.path.join(download_path, 'embeddings.tsv'), 'w', newline='') as f_output:\n",
        "        tsv_output = csv.writer(f_output, delimiter='\\t')\n",
        "        tsv_output.writerows(latent_representations)\n",
        "    # Write corresdonding filenames\n",
        "    with open(os.path.join(download_path, 'embedding-filenames.tsv'), 'a') as f_output:\n",
        "        f_output.seek(0)\n",
        "        f_output.truncate()\n",
        "        for data in dsfilenames:\n",
        "            f_output.write(data)\n",
        "            f_output.write('\\n')\n",
        "    print(\"Embeddings saved!!\")\n",
        "    return latent_representations\n",
        "\n",
        "## Driver coder\n",
        "\n",
        "if musicae == None:\n",
        "    musicae = VAE.load(MODEL_PATH)\n",
        "\n",
        "save_embeddings(musicae, dsfilenames, MODEL_PATH)\n",
        "\n",
        "# reconstructed_images, _ = autoencoder.reconstruct(np.array(list(dataset.values())))\n",
        "# plot_reconstructed_images(sample_images, reconstructed_images)"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:2325: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
            "  warnings.warn('`Model.state_updates` will be removed in a future version. '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Embeddings saved!!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.94976586,  3.589117  , -5.6696005 , ..., -2.1024005 ,\n",
              "        -0.52547985,  0.53528875],\n",
              "       [-0.39703095,  0.09528753, -1.1663307 , ...,  3.14496   ,\n",
              "        -3.7484055 , -0.44704148],\n",
              "       [-3.3417277 ,  4.7321954 ,  1.7879758 , ..., -2.5602322 ,\n",
              "         0.777218  ,  2.006522  ],\n",
              "       ...,\n",
              "       [ 0.72897494,  4.4406204 ,  3.682107  , ...,  3.967624  ,\n",
              "         7.013556  ,  1.5005401 ],\n",
              "       [-3.957868  , -0.02090748, -1.6777387 , ...,  6.1119556 ,\n",
              "         2.5452788 ,  6.7693944 ],\n",
              "       [-4.885683  ,  2.5795383 , -1.1673104 , ...,  8.482125  ,\n",
              "         2.1263878 ,  3.0961082 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cuDjZV1Fxs9d"
      },
      "source": [
        "### test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IrMv7xaP5O8l"
      },
      "source": [
        "inputs = Input(shape=(32,))\n",
        "mid = Dense(2)(inputs)\n",
        "outputs = Dense(1)(mid)\n",
        "model = Model(inputs, outputs)\n",
        "model.compile(optimizer=\"adam\", loss=\"mean_squared_error\")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Z2OQsDi6kj0",
        "outputId": "31fabcf3-93e9-4e17-900a-1c74633c9e1a"
      },
      "source": [
        "test_input = np.random.random((128, 32))\n",
        "test_target = np.random.random((128, 1))\n",
        "model.fit(test_input, test_target)\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train on 128 samples\n",
            "128/128 [==============================] - 1s 11ms/sample - loss: 0.2531\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f7103de1bd0>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OusbhJJo6oCc",
        "outputId": "2ed42b65-e7be-4a1b-aec9-c6120c785b27"
      },
      "source": [
        "model.save('/content/drive/MyDrive/Music/VAE/ICASSP/tmp')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Music/VAE/ICASSP/tmp/assets\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1z6Oj79epepj",
        "outputId": "46f07952-6c14-41af-d166-6c13057fc078"
      },
      "source": [
        "print(model.outputs)\n",
        "print(model.layers[1].get_config())\n",
        "print(model.layers[1].get_weights())\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[<tf.Tensor 'dense_1/BiasAdd:0' shape=(None, 1) dtype=float32>]\n",
            "{'name': 'dense', 'trainable': True, 'dtype': 'float32', 'units': 2, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}\n",
            "[array([[-0.4075677 ,  0.03736704],\n",
            "       [ 0.33599454,  0.01744784],\n",
            "       [-0.05095035, -0.09282304],\n",
            "       [-0.01146705,  0.0266833 ],\n",
            "       [-0.2184882 , -0.38033777],\n",
            "       [-0.41455442, -0.4095814 ],\n",
            "       [ 0.40297043,  0.20315753],\n",
            "       [-0.12497225, -0.23888613],\n",
            "       [ 0.08572217,  0.20766038],\n",
            "       [ 0.04210095, -0.14410129],\n",
            "       [-0.02147608, -0.02591982],\n",
            "       [ 0.27241927,  0.28874815],\n",
            "       [-0.3938247 ,  0.4193524 ],\n",
            "       [ 0.3027814 , -0.3117693 ],\n",
            "       [ 0.13342443,  0.24553962],\n",
            "       [-0.18907784,  0.00208663],\n",
            "       [-0.40545473, -0.21231669],\n",
            "       [ 0.26950115,  0.42039087],\n",
            "       [-0.19123167, -0.01983401],\n",
            "       [ 0.01332134,  0.29213002],\n",
            "       [ 0.33358467, -0.2966309 ],\n",
            "       [-0.18340905, -0.29907537],\n",
            "       [ 0.37802315,  0.00852278],\n",
            "       [-0.08358875, -0.20628898],\n",
            "       [-0.15099894,  0.19588183],\n",
            "       [-0.15138943,  0.1879225 ],\n",
            "       [ 0.35929438,  0.2728458 ],\n",
            "       [-0.3536343 ,  0.37920487],\n",
            "       [-0.41077444,  0.18115307],\n",
            "       [ 0.39061818,  0.05495013],\n",
            "       [-0.2563238 , -0.16152485],\n",
            "       [-0.25270125,  0.33373627]], dtype=float32), array([0.00388095, 0.00388384], dtype=float32)]\n"
          ]
        }
      ]
    }
  ]
}