QAT-Resnet18-train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import tensorflow.keras as keras\n",
    "import tensorflow_model_optimization as tfmot\n",
    "from tensorflow.keras.layers import Dense, Conv2D, BatchNormalization, Activation\n",
    "from tensorflow.keras.layers import AveragePooling2D, Input, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras import backend as K\n",
    "from art.defences import AdversarialTrainer\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "\n",
    "from art.attacks import FastGradientMethod, ProjectedGradientDescent, PixelAttack\n",
    "from art.classifiers import KerasClassifier\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=l2(1e-4))\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        #if batch_normalization:\n",
    "        #    x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        #if batch_normalization:\n",
    "        #    x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v2(input_shape, depth, num_classes=10):\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = False\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    #x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "y_train shape: (50000, 1)\n",
      "Learning rate:  0.001\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 32, 32, 16)   64          conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 32, 32, 16)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   272         activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   2320        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 64)   1088        activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 64)   1088        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 32, 32, 64)   0           conv2d_4[0][0]                   \n",
      "                                                                 conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 64)   256         add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 64)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 16)   1040        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 16)   64          conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 16)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   2320        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 32, 32, 64)   1088        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 64)   0           add[0][0]                        \n",
      "                                                                 conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 64)   256         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 64)   0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 16, 16, 64)   4160        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 16, 16, 64)   256         conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 16, 16, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 64)   36928       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 16, 16, 64)   256         conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 64)   0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 128)  8320        add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 128)  8320        activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 16, 16, 128)  0           conv2d_11[0][0]                  \n",
      "                                                                 conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 16, 16, 128)  512         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 128)  0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 64)   8256        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 64)   256         conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 16, 16, 64)   0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 64)   36928       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 64)   256         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 16, 16, 64)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 16, 16, 128)  8320        activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 128)  0           add_2[0][0]                      \n",
      "                                                                 conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 128)  512         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 16, 16, 128)  0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 8, 8, 128)    16512       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 8, 8, 128)    512         conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 8, 8, 128)    0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 8, 8, 128)    147584      activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 128)    512         conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 8, 8, 128)    0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 8, 8, 256)    33024       add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 8, 8, 256)    33024       activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 8, 8, 256)    0           conv2d_18[0][0]                  \n",
      "                                                                 conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 256)    1024        add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 8, 8, 256)    0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 8, 8, 128)    32896       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 8, 8, 128)    512         conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 8, 8, 128)    0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 128)    147584      activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 8, 8, 128)    512         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 8, 8, 128)    0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 8, 8, 256)    33024       activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 8, 8, 256)    0           add_4[0][0]                      \n",
      "                                                                 conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 8, 8, 256)    1024        add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 8, 8, 256)    0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, 1, 1, 256)    0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 256)          0           average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 10)           2570        flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 574,090\n",
      "Trainable params: 570,602\n",
      "Non-trainable params: 3,488\n",
      "__________________________________________________________________________________________________\n",
      "ResNet20v2\n",
      "Using real-time data augmentation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-7067b6c78ff2>:147: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use Model.fit, which supports generators.\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train for 391 steps, validate on 10000 samples\n",
      "Learning rate:  0.001\n",
      "Epoch 1/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 1.8485 - accuracy: 0.4676WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 24s 62ms/step - loss: 1.8464 - accuracy: 0.4681 - val_loss: 1.6336 - val_accuracy: 0.5273\n",
      "Learning rate:  0.001\n",
      "Epoch 2/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.4275 - accuracy: 0.6049WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 1.4274 - accuracy: 0.6049 - val_loss: 1.6791 - val_accuracy: 0.5482\n",
      "Learning rate:  0.001\n",
      "Epoch 3/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.2294 - accuracy: 0.6690WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 1.2291 - accuracy: 0.6691 - val_loss: 1.2182 - val_accuracy: 0.6724\n",
      "Learning rate:  0.001\n",
      "Epoch 4/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.1047 - accuracy: 0.7102WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 1.1044 - accuracy: 0.7103 - val_loss: 1.3946 - val_accuracy: 0.6401\n",
      "Learning rate:  0.001\n",
      "Epoch 5/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.0094 - accuracy: 0.7400WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 1.0093 - accuracy: 0.7400 - val_loss: 1.0954 - val_accuracy: 0.7212\n",
      "Learning rate:  0.001\n",
      "Epoch 6/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.9444 - accuracy: 0.7606WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.9447 - accuracy: 0.7605 - val_loss: 1.0155 - val_accuracy: 0.7358\n",
      "Learning rate:  0.001\n",
      "Epoch 7/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.8841 - accuracy: 0.7804WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.8844 - accuracy: 0.7803 - val_loss: 1.2181 - val_accuracy: 0.6835\n",
      "Learning rate:  0.001\n",
      "Epoch 8/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8405 - accuracy: 0.7936WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.8409 - accuracy: 0.7935 - val_loss: 1.1341 - val_accuracy: 0.7104\n",
      "Learning rate:  0.001\n",
      "Epoch 9/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8088 - accuracy: 0.8030WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.8085 - accuracy: 0.8032 - val_loss: 0.9129 - val_accuracy: 0.7709\n",
      "Learning rate:  0.001\n",
      "Epoch 10/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.7788 - accuracy: 0.8121WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.7786 - accuracy: 0.8122 - val_loss: 0.9409 - val_accuracy: 0.7690\n",
      "Learning rate:  0.001\n",
      "Epoch 11/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.7470 - accuracy: 0.8217WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.7477 - accuracy: 0.8215 - val_loss: 0.8208 - val_accuracy: 0.7990\n",
      "Learning rate:  0.001\n",
      "Epoch 12/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.7273 - accuracy: 0.8249WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.7272 - accuracy: 0.8250 - val_loss: 0.8363 - val_accuracy: 0.7971\n",
      "Learning rate:  0.001\n",
      "Epoch 13/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.7023 - accuracy: 0.8343WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.7024 - accuracy: 0.8342 - val_loss: 1.1483 - val_accuracy: 0.7285\n",
      "Learning rate:  0.001\n",
      "Epoch 14/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6834 - accuracy: 0.8397WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.6836 - accuracy: 0.8396 - val_loss: 0.9929 - val_accuracy: 0.7556\n",
      "Learning rate:  0.001\n",
      "Epoch 15/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6689 - accuracy: 0.8446WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.6691 - accuracy: 0.8446 - val_loss: 0.8398 - val_accuracy: 0.7947\n",
      "Learning rate:  0.001\n",
      "Epoch 16/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6554 - accuracy: 0.8477WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.6554 - accuracy: 0.8477 - val_loss: 0.8738 - val_accuracy: 0.7831\n",
      "Learning rate:  0.001\n",
      "Epoch 17/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6348 - accuracy: 0.8542WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.6347 - accuracy: 0.8542 - val_loss: 0.8939 - val_accuracy: 0.7769\n",
      "Learning rate:  0.001\n",
      "Epoch 18/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6292 - accuracy: 0.8554WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.6283 - accuracy: 0.8558 - val_loss: 0.7534 - val_accuracy: 0.8202\n",
      "Learning rate:  0.001\n",
      "Epoch 19/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6145 - accuracy: 0.8620WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.6147 - accuracy: 0.8620 - val_loss: 0.9484 - val_accuracy: 0.7708\n",
      "Learning rate:  0.001\n",
      "Epoch 20/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.6017 - accuracy: 0.8646WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.6015 - accuracy: 0.8646 - val_loss: 0.8926 - val_accuracy: 0.7710\n",
      "Learning rate:  0.001\n",
      "Epoch 21/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5975 - accuracy: 0.8653WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 58ms/step - loss: 0.5976 - accuracy: 0.8652 - val_loss: 0.7375 - val_accuracy: 0.8283\n",
      "Learning rate:  0.001\n",
      "Epoch 22/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5886 - accuracy: 0.8686WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 60ms/step - loss: 0.5882 - accuracy: 0.8688 - val_loss: 0.6957 - val_accuracy: 0.8323\n",
      "Learning rate:  0.001\n",
      "Epoch 23/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5753 - accuracy: 0.8722WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.5756 - accuracy: 0.8721 - val_loss: 0.8942 - val_accuracy: 0.7928\n",
      "Learning rate:  0.001\n",
      "Epoch 24/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5676 - accuracy: 0.8746WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.5674 - accuracy: 0.8747 - val_loss: 0.7113 - val_accuracy: 0.8325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 25/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.5573 - accuracy: 0.8778WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.5576 - accuracy: 0.8775 - val_loss: 0.8276 - val_accuracy: 0.8081\n",
      "Learning rate:  0.001\n",
      "Epoch 26/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.5551 - accuracy: 0.8782WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 59ms/step - loss: 0.5553 - accuracy: 0.8781 - val_loss: 0.8703 - val_accuracy: 0.7887\n",
      "Learning rate:  0.001\n",
      "Epoch 27/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5458 - accuracy: 0.8824WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.5459 - accuracy: 0.8824 - val_loss: 0.8449 - val_accuracy: 0.8098\n",
      "Learning rate:  0.001\n",
      "Epoch 28/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5363 - accuracy: 0.8840WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.5364 - accuracy: 0.8839 - val_loss: 0.7391 - val_accuracy: 0.8256\n",
      "Learning rate:  0.001\n",
      "Epoch 29/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.5281 - accuracy: 0.8867WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.5282 - accuracy: 0.8867 - val_loss: 0.8423 - val_accuracy: 0.7986\n",
      "Learning rate:  0.001\n",
      "Epoch 30/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.5254 - accuracy: 0.8859WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.5254 - accuracy: 0.8860 - val_loss: 1.0731 - val_accuracy: 0.7372\n",
      "Learning rate:  0.001\n",
      "Epoch 31/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5150 - accuracy: 0.8900WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.5148 - accuracy: 0.8900 - val_loss: 0.7522 - val_accuracy: 0.8313\n",
      "Learning rate:  0.001\n",
      "Epoch 32/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5134 - accuracy: 0.8918 ETA: 1s -WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.5136 - accuracy: 0.8918 - val_loss: 0.7287 - val_accuracy: 0.8434\n",
      "Learning rate:  0.001\n",
      "Epoch 33/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5057 - accuracy: 0.8946WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.5058 - accuracy: 0.8946 - val_loss: 0.7798 - val_accuracy: 0.8031\n",
      "Learning rate:  0.001\n",
      "Epoch 34/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5069 - accuracy: 0.8942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.5067 - accuracy: 0.8943 - val_loss: 0.7430 - val_accuracy: 0.8277\n",
      "Learning rate:  0.001\n",
      "Epoch 35/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.5006 - accuracy: 0.8948WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.5007 - accuracy: 0.8947 - val_loss: 0.6888 - val_accuracy: 0.8452\n",
      "Learning rate:  0.001\n",
      "Epoch 36/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4929 - accuracy: 0.8984WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 24s 62ms/step - loss: 0.4930 - accuracy: 0.8984 - val_loss: 0.7466 - val_accuracy: 0.8259\n",
      "Learning rate:  0.001\n",
      "Epoch 37/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4904 - accuracy: 0.8981WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4904 - accuracy: 0.8981 - val_loss: 0.8555 - val_accuracy: 0.8075\n",
      "Learning rate:  0.001\n",
      "Epoch 38/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4863 - accuracy: 0.9019WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4863 - accuracy: 0.9018 - val_loss: 0.7218 - val_accuracy: 0.8288\n",
      "Learning rate:  0.001\n",
      "Epoch 39/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4836 - accuracy: 0.9002WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 51ms/step - loss: 0.4834 - accuracy: 0.9001 - val_loss: 0.7345 - val_accuracy: 0.8333\n",
      "Learning rate:  0.001\n",
      "Epoch 40/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4871 - accuracy: 0.9001WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.4872 - accuracy: 0.9001 - val_loss: 0.6811 - val_accuracy: 0.8415\n",
      "Learning rate:  0.001\n",
      "Epoch 41/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4712 - accuracy: 0.9052WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.4713 - accuracy: 0.9051 - val_loss: 0.6968 - val_accuracy: 0.8387\n",
      "Learning rate:  0.001\n",
      "Epoch 42/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4739 - accuracy: 0.9049WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4737 - accuracy: 0.9049 - val_loss: 0.7385 - val_accuracy: 0.8347\n",
      "Learning rate:  0.001\n",
      "Epoch 43/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4666 - accuracy: 0.9076WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.4664 - accuracy: 0.9077 - val_loss: 0.7039 - val_accuracy: 0.8469\n",
      "Learning rate:  0.001\n",
      "Epoch 44/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4641 - accuracy: 0.9081WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4643 - accuracy: 0.9080 - val_loss: 0.6791 - val_accuracy: 0.8492\n",
      "Learning rate:  0.001\n",
      "Epoch 45/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4678 - accuracy: 0.9059WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 24s 62ms/step - loss: 0.4678 - accuracy: 0.9059 - val_loss: 0.7537 - val_accuracy: 0.8214\n",
      "Learning rate:  0.001\n",
      "Epoch 46/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4576 - accuracy: 0.9104WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4577 - accuracy: 0.9103 - val_loss: 0.6910 - val_accuracy: 0.8482\n",
      "Learning rate:  0.001\n",
      "Epoch 47/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4592 - accuracy: 0.9089WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4593 - accuracy: 0.9089 - val_loss: 0.6697 - val_accuracy: 0.8501\n",
      "Learning rate:  0.001\n",
      "Epoch 48/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4523 - accuracy: 0.9119WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4522 - accuracy: 0.9119 - val_loss: 0.7384 - val_accuracy: 0.8344\n",
      "Learning rate:  0.001\n",
      "Epoch 49/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4527 - accuracy: 0.9119WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4530 - accuracy: 0.9118 - val_loss: 0.6961 - val_accuracy: 0.8446\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 50/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4502 - accuracy: 0.9127WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4502 - accuracy: 0.9128 - val_loss: 0.7409 - val_accuracy: 0.8310\n",
      "Learning rate:  0.001\n",
      "Epoch 51/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4519 - accuracy: 0.9113WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4520 - accuracy: 0.9114 - val_loss: 0.7779 - val_accuracy: 0.8272\n",
      "Learning rate:  0.001\n",
      "Epoch 52/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4414 - accuracy: 0.9157WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4413 - accuracy: 0.9157 - val_loss: 0.7755 - val_accuracy: 0.8161\n",
      "Learning rate:  0.001\n",
      "Epoch 53/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4470 - accuracy: 0.9140WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4475 - accuracy: 0.9139 - val_loss: 0.6228 - val_accuracy: 0.8605\n",
      "Learning rate:  0.001\n",
      "Epoch 54/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4369 - accuracy: 0.9171WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 51ms/step - loss: 0.4369 - accuracy: 0.9171 - val_loss: 0.8073 - val_accuracy: 0.8198\n",
      "Learning rate:  0.001\n",
      "Epoch 55/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4350 - accuracy: 0.9172WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.4350 - accuracy: 0.9173 - val_loss: 0.6788 - val_accuracy: 0.8533\n",
      "Learning rate:  0.001\n",
      "Epoch 56/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4340 - accuracy: 0.9181WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.4338 - accuracy: 0.9181 - val_loss: 0.7541 - val_accuracy: 0.8332\n",
      "Learning rate:  0.001\n",
      "Epoch 57/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4347 - accuracy: 0.9177WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.4352 - accuracy: 0.9175 - val_loss: 0.7280 - val_accuracy: 0.8408\n",
      "Learning rate:  0.001\n",
      "Epoch 58/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4325 - accuracy: 0.9192WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4326 - accuracy: 0.9192 - val_loss: 0.8535 - val_accuracy: 0.8157\n",
      "Learning rate:  0.001\n",
      "Epoch 59/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4338 - accuracy: 0.9174WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4336 - accuracy: 0.9175 - val_loss: 0.7932 - val_accuracy: 0.8173\n",
      "Learning rate:  0.001\n",
      "Epoch 60/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4237 - accuracy: 0.9228WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4237 - accuracy: 0.9229 - val_loss: 0.7655 - val_accuracy: 0.8397\n",
      "Learning rate:  0.001\n",
      "Epoch 61/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4238 - accuracy: 0.9215WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4240 - accuracy: 0.9214 - val_loss: 0.8554 - val_accuracy: 0.8096\n",
      "Learning rate:  0.001\n",
      "Epoch 62/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4242 - accuracy: 0.9212WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4244 - accuracy: 0.9211 - val_loss: 0.7577 - val_accuracy: 0.8345\n",
      "Learning rate:  0.001\n",
      "Epoch 63/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4240 - accuracy: 0.9221WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.4239 - accuracy: 0.9222 - val_loss: 0.6262 - val_accuracy: 0.8628\n",
      "Learning rate:  0.001\n",
      "Epoch 64/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4204 - accuracy: 0.9231WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4203 - accuracy: 0.9230 - val_loss: 0.8554 - val_accuracy: 0.8144\n",
      "Learning rate:  0.001\n",
      "Epoch 65/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4218 - accuracy: 0.9226WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4219 - accuracy: 0.9226 - val_loss: 0.6297 - val_accuracy: 0.8615\n",
      "Learning rate:  0.001\n",
      "Epoch 66/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4147 - accuracy: 0.9244WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.4148 - accuracy: 0.9244 - val_loss: 0.8086 - val_accuracy: 0.8307\n",
      "Learning rate:  0.001\n",
      "Epoch 67/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4192 - accuracy: 0.9223WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4197 - accuracy: 0.9222 - val_loss: 0.6914 - val_accuracy: 0.8543\n",
      "Learning rate:  0.001\n",
      "Epoch 68/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4137 - accuracy: 0.9238WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.4136 - accuracy: 0.9239 - val_loss: 0.7276 - val_accuracy: 0.8427\n",
      "Learning rate:  0.001\n",
      "Epoch 69/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4114 - accuracy: 0.9244WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.4116 - accuracy: 0.9244 - val_loss: 0.6811 - val_accuracy: 0.8485\n",
      "Learning rate:  0.001\n",
      "Epoch 70/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4126 - accuracy: 0.9250WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.4127 - accuracy: 0.9250 - val_loss: 0.9683 - val_accuracy: 0.8013\n",
      "Learning rate:  0.001\n",
      "Epoch 71/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4085 - accuracy: 0.9273WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.4089 - accuracy: 0.9271 - val_loss: 0.6907 - val_accuracy: 0.8581\n",
      "Learning rate:  0.001\n",
      "Epoch 72/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4085 - accuracy: 0.9266WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4086 - accuracy: 0.9266 - val_loss: 0.7364 - val_accuracy: 0.8400\n",
      "Learning rate:  0.001\n",
      "Epoch 73/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4109 - accuracy: 0.9248WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4108 - accuracy: 0.9249 - val_loss: 0.7291 - val_accuracy: 0.8469\n",
      "Learning rate:  0.001\n",
      "Epoch 74/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4049 - accuracy: 0.9267WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4050 - accuracy: 0.9267 - val_loss: 0.7957 - val_accuracy: 0.8312\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 75/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4026 - accuracy: 0.9285WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4026 - accuracy: 0.9284 - val_loss: 0.7501 - val_accuracy: 0.8365\n",
      "Learning rate:  0.001\n",
      "Epoch 76/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4064 - accuracy: 0.9263 ETA: 0s - loss: 0.4048 - acWARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.4065 - accuracy: 0.9262 - val_loss: 0.6396 - val_accuracy: 0.8552\n",
      "Learning rate:  0.001\n",
      "Epoch 77/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4023 - accuracy: 0.9279WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 60ms/step - loss: 0.4025 - accuracy: 0.9278 - val_loss: 0.7582 - val_accuracy: 0.8423\n",
      "Learning rate:  0.001\n",
      "Epoch 78/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.4025 - accuracy: 0.9287WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.4023 - accuracy: 0.9287 - val_loss: 0.6431 - val_accuracy: 0.8634\n",
      "Learning rate:  0.001\n",
      "Epoch 79/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4049 - accuracy: 0.9280WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.4049 - accuracy: 0.9280 - val_loss: 0.8589 - val_accuracy: 0.8229\n",
      "Learning rate:  0.001\n",
      "Epoch 80/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.3985 - accuracy: 0.9297WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.3987 - accuracy: 0.9296 - val_loss: 0.7311 - val_accuracy: 0.8448\n",
      "Learning rate:  0.001\n",
      "Epoch 81/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.3992 - accuracy: 0.9290WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.3990 - accuracy: 0.9291 - val_loss: 0.7332 - val_accuracy: 0.8477\n",
      "Learning rate:  0.0001\n",
      "Epoch 82/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.3283 - accuracy: 0.9557WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 51ms/step - loss: 0.3283 - accuracy: 0.9556 - val_loss: 0.5046 - val_accuracy: 0.9047\n",
      "Learning rate:  0.0001\n",
      "Epoch 83/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.3007 - accuracy: 0.9658WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.3006 - accuracy: 0.9658 - val_loss: 0.5007 - val_accuracy: 0.9057\n",
      "Learning rate:  0.0001\n",
      "Epoch 84/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2900 - accuracy: 0.9693WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.2900 - accuracy: 0.9693 - val_loss: 0.5003 - val_accuracy: 0.9058\n",
      "Learning rate:  0.0001\n",
      "Epoch 85/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2813 - accuracy: 0.9709WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.2813 - accuracy: 0.9709 - val_loss: 0.5016 - val_accuracy: 0.9068\n",
      "Learning rate:  0.0001\n",
      "Epoch 86/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2730 - accuracy: 0.9736WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2730 - accuracy: 0.9735 - val_loss: 0.4975 - val_accuracy: 0.9076\n",
      "Learning rate:  0.0001\n",
      "Epoch 87/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2651 - accuracy: 0.9749WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.2651 - accuracy: 0.9749 - val_loss: 0.5048 - val_accuracy: 0.9058\n",
      "Learning rate:  0.0001\n",
      "Epoch 88/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2611 - accuracy: 0.9767WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2611 - accuracy: 0.9767 - val_loss: 0.5008 - val_accuracy: 0.9081\n",
      "Learning rate:  0.0001\n",
      "Epoch 89/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2561 - accuracy: 0.9765WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.2560 - accuracy: 0.9766 - val_loss: 0.5059 - val_accuracy: 0.9071\n",
      "Learning rate:  0.0001\n",
      "Epoch 90/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2534 - accuracy: 0.9765WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.2533 - accuracy: 0.9765 - val_loss: 0.4929 - val_accuracy: 0.9097\n",
      "Learning rate:  0.0001\n",
      "Epoch 91/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2448 - accuracy: 0.9793WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.2449 - accuracy: 0.9793 - val_loss: 0.4926 - val_accuracy: 0.9113\n",
      "Learning rate:  0.0001\n",
      "Epoch 92/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2435 - accuracy: 0.9795WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.2435 - accuracy: 0.9795 - val_loss: 0.5009 - val_accuracy: 0.9088\n",
      "Learning rate:  0.0001\n",
      "Epoch 93/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2392 - accuracy: 0.9792WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.2393 - accuracy: 0.9791 - val_loss: 0.5049 - val_accuracy: 0.9110\n",
      "Learning rate:  0.0001\n",
      "Epoch 94/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2350 - accuracy: 0.9813WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.2349 - accuracy: 0.9814 - val_loss: 0.5019 - val_accuracy: 0.9087\n",
      "Learning rate:  0.0001\n",
      "Epoch 95/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2326 - accuracy: 0.9812WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.2327 - accuracy: 0.9811 - val_loss: 0.4978 - val_accuracy: 0.9117\n",
      "Learning rate:  0.0001\n",
      "Epoch 96/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2272 - accuracy: 0.9821WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.2272 - accuracy: 0.9821 - val_loss: 0.4931 - val_accuracy: 0.9099\n",
      "Learning rate:  0.0001\n",
      "Epoch 97/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2240 - accuracy: 0.9828WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.2240 - accuracy: 0.9829 - val_loss: 0.5096 - val_accuracy: 0.9110\n",
      "Learning rate:  0.0001\n",
      "Epoch 98/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2208 - accuracy: 0.9832WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2208 - accuracy: 0.9832 - val_loss: 0.5058 - val_accuracy: 0.9109\n",
      "Learning rate:  0.0001\n",
      "Epoch 99/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2171 - accuracy: 0.9837WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2170 - accuracy: 0.9837 - val_loss: 0.4998 - val_accuracy: 0.9092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.0001\n",
      "Epoch 100/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2148 - accuracy: 0.9848WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.2147 - accuracy: 0.9848 - val_loss: 0.4981 - val_accuracy: 0.9116\n",
      "Learning rate:  0.0001\n",
      "Epoch 101/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2129 - accuracy: 0.9850WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.2130 - accuracy: 0.9850 - val_loss: 0.5182 - val_accuracy: 0.9065\n",
      "Learning rate:  0.0001\n",
      "Epoch 102/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2085 - accuracy: 0.9857WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.2085 - accuracy: 0.9857 - val_loss: 0.5236 - val_accuracy: 0.9077\n",
      "Learning rate:  0.0001\n",
      "Epoch 103/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2063 - accuracy: 0.9862WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2062 - accuracy: 0.9862 - val_loss: 0.5042 - val_accuracy: 0.9090\n",
      "Learning rate:  0.0001\n",
      "Epoch 104/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2041 - accuracy: 0.9863WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2040 - accuracy: 0.9864 - val_loss: 0.5133 - val_accuracy: 0.9087\n",
      "Learning rate:  0.0001\n",
      "Epoch 105/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.2026 - accuracy: 0.9863WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.2028 - accuracy: 0.9862 - val_loss: 0.5186 - val_accuracy: 0.9069\n",
      "Learning rate:  0.0001\n",
      "Epoch 106/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1978 - accuracy: 0.9878WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.1978 - accuracy: 0.9878 - val_loss: 0.5151 - val_accuracy: 0.9061\n",
      "Learning rate:  0.0001\n",
      "Epoch 107/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1971 - accuracy: 0.9874WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.1973 - accuracy: 0.9873 - val_loss: 0.5102 - val_accuracy: 0.9098\n",
      "Learning rate:  0.0001\n",
      "Epoch 108/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1946 - accuracy: 0.9877WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.1946 - accuracy: 0.9876 - val_loss: 0.5113 - val_accuracy: 0.9072\n",
      "Learning rate:  0.0001\n",
      "Epoch 109/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1933 - accuracy: 0.9877WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 64ms/step - loss: 0.1933 - accuracy: 0.9877 - val_loss: 0.5128 - val_accuracy: 0.9057\n",
      "Learning rate:  0.0001\n",
      "Epoch 110/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1905 - accuracy: 0.9879WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1905 - accuracy: 0.9878 - val_loss: 0.5125 - val_accuracy: 0.9093\n",
      "Learning rate:  0.0001\n",
      "Epoch 111/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1888 - accuracy: 0.9885WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1887 - accuracy: 0.9885 - val_loss: 0.5111 - val_accuracy: 0.9077\n",
      "Learning rate:  0.0001\n",
      "Epoch 112/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1859 - accuracy: 0.9888WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.1859 - accuracy: 0.9888 - val_loss: 0.5148 - val_accuracy: 0.9068\n",
      "Learning rate:  0.0001\n",
      "Epoch 113/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1854 - accuracy: 0.9889WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1855 - accuracy: 0.9889 - val_loss: 0.5114 - val_accuracy: 0.9085\n",
      "Learning rate:  0.0001\n",
      "Epoch 114/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1820 - accuracy: 0.9897WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1820 - accuracy: 0.9897 - val_loss: 0.5176 - val_accuracy: 0.9071\n",
      "Learning rate:  0.0001\n",
      "Epoch 115/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1785 - accuracy: 0.9906WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1786 - accuracy: 0.9905 - val_loss: 0.5156 - val_accuracy: 0.9088\n",
      "Learning rate:  0.0001\n",
      "Epoch 116/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1799 - accuracy: 0.9893WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1800 - accuracy: 0.9893 - val_loss: 0.5202 - val_accuracy: 0.9082\n",
      "Learning rate:  0.0001\n",
      "Epoch 117/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1773 - accuracy: 0.9889WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1775 - accuracy: 0.9889 - val_loss: 0.5235 - val_accuracy: 0.9066\n",
      "Learning rate:  0.0001\n",
      "Epoch 118/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1747 - accuracy: 0.9906WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1748 - accuracy: 0.9906 - val_loss: 0.5356 - val_accuracy: 0.9078\n",
      "Learning rate:  0.0001\n",
      "Epoch 119/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1762 - accuracy: 0.9893WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 55ms/step - loss: 0.1762 - accuracy: 0.9893 - val_loss: 0.5241 - val_accuracy: 0.9076\n",
      "Learning rate:  0.0001\n",
      "Epoch 120/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1729 - accuracy: 0.9901WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1729 - accuracy: 0.9901 - val_loss: 0.5226 - val_accuracy: 0.9083\n",
      "Learning rate:  0.0001\n",
      "Epoch 121/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1739 - accuracy: 0.9894WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1738 - accuracy: 0.9895 - val_loss: 0.5206 - val_accuracy: 0.9077\n",
      "Learning rate:  1e-05\n",
      "Epoch 122/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1665 - accuracy: 0.9922WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1666 - accuracy: 0.9922 - val_loss: 0.5100 - val_accuracy: 0.9100\n",
      "Learning rate:  1e-05\n",
      "Epoch 123/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1647 - accuracy: 0.9933WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1648 - accuracy: 0.9932 - val_loss: 0.5057 - val_accuracy: 0.9103\n",
      "Learning rate:  1e-05\n",
      "Epoch 124/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1639 - accuracy: 0.9931WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.1639 - accuracy: 0.9931 - val_loss: 0.5056 - val_accuracy: 0.9120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  1e-05\n",
      "Epoch 125/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1640 - accuracy: 0.9931WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1640 - accuracy: 0.9931 - val_loss: 0.5034 - val_accuracy: 0.9125\n",
      "Learning rate:  1e-05\n",
      "Epoch 126/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1625 - accuracy: 0.9934WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1625 - accuracy: 0.9934 - val_loss: 0.5037 - val_accuracy: 0.9122\n",
      "Learning rate:  1e-05\n",
      "Epoch 127/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1629 - accuracy: 0.9935WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1629 - accuracy: 0.9935 - val_loss: 0.5053 - val_accuracy: 0.9125\n",
      "Learning rate:  1e-05\n",
      "Epoch 128/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1621 - accuracy: 0.9935WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1620 - accuracy: 0.9936 - val_loss: 0.5032 - val_accuracy: 0.9122\n",
      "Learning rate:  1e-05\n",
      "Epoch 129/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1609 - accuracy: 0.9942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1609 - accuracy: 0.9941 - val_loss: 0.5024 - val_accuracy: 0.9122\n",
      "Learning rate:  1e-05\n",
      "Epoch 130/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1620 - accuracy: 0.9932WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.1619 - accuracy: 0.9933 - val_loss: 0.5020 - val_accuracy: 0.9136\n",
      "Learning rate:  1e-05\n",
      "Epoch 131/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1609 - accuracy: 0.9936WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1609 - accuracy: 0.9936 - val_loss: 0.5011 - val_accuracy: 0.9136\n",
      "Learning rate:  1e-05\n",
      "Epoch 132/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1593 - accuracy: 0.9943WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1593 - accuracy: 0.9943 - val_loss: 0.5005 - val_accuracy: 0.9142\n",
      "Learning rate:  1e-05\n",
      "Epoch 133/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1598 - accuracy: 0.9939WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1598 - accuracy: 0.9939 - val_loss: 0.5036 - val_accuracy: 0.9137\n",
      "Learning rate:  1e-05\n",
      "Epoch 134/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1590 - accuracy: 0.9943WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1590 - accuracy: 0.9943 - val_loss: 0.5031 - val_accuracy: 0.9149\n",
      "Learning rate:  1e-05\n",
      "Epoch 135/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1600 - accuracy: 0.9943WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1600 - accuracy: 0.9943 - val_loss: 0.5053 - val_accuracy: 0.9127\n",
      "Learning rate:  1e-05\n",
      "Epoch 136/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1589 - accuracy: 0.9945WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1589 - accuracy: 0.9945 - val_loss: 0.5038 - val_accuracy: 0.9130\n",
      "Learning rate:  1e-05\n",
      "Epoch 137/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1586 - accuracy: 0.9942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.1585 - accuracy: 0.9942 - val_loss: 0.5041 - val_accuracy: 0.9129\n",
      "Learning rate:  1e-05\n",
      "Epoch 138/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1581 - accuracy: 0.9945WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1580 - accuracy: 0.9946 - val_loss: 0.5062 - val_accuracy: 0.9127\n",
      "Learning rate:  1e-05\n",
      "Epoch 139/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1583 - accuracy: 0.9942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1583 - accuracy: 0.9942 - val_loss: 0.5025 - val_accuracy: 0.9137\n",
      "Learning rate:  1e-05\n",
      "Epoch 140/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1566 - accuracy: 0.9951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1567 - accuracy: 0.9950 - val_loss: 0.5054 - val_accuracy: 0.9125\n",
      "Learning rate:  1e-05\n",
      "Epoch 141/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1570 - accuracy: 0.9949WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1570 - accuracy: 0.9949 - val_loss: 0.5040 - val_accuracy: 0.9122\n",
      "Learning rate:  1e-05\n",
      "Epoch 142/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1578 - accuracy: 0.9943WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 55ms/step - loss: 0.1577 - accuracy: 0.9944 - val_loss: 0.5039 - val_accuracy: 0.9116\n",
      "Learning rate:  1e-05\n",
      "Epoch 143/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1566 - accuracy: 0.9946WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 60ms/step - loss: 0.1566 - accuracy: 0.9946 - val_loss: 0.5031 - val_accuracy: 0.9132\n",
      "Learning rate:  1e-05\n",
      "Epoch 144/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1569 - accuracy: 0.9942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 65ms/step - loss: 0.1569 - accuracy: 0.9942 - val_loss: 0.5040 - val_accuracy: 0.9129\n",
      "Learning rate:  1e-05\n",
      "Epoch 145/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1560 - accuracy: 0.9949WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1561 - accuracy: 0.9949 - val_loss: 0.5033 - val_accuracy: 0.9132\n",
      "Learning rate:  1e-05\n",
      "Epoch 146/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1552 - accuracy: 0.9951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 65ms/step - loss: 0.1552 - accuracy: 0.9950 - val_loss: 0.5043 - val_accuracy: 0.9109\n",
      "Learning rate:  1e-05\n",
      "Epoch 147/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1566 - accuracy: 0.9946WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1565 - accuracy: 0.9947 - val_loss: 0.5056 - val_accuracy: 0.9120\n",
      "Learning rate:  1e-05\n",
      "Epoch 148/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1561 - accuracy: 0.9943WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1561 - accuracy: 0.9943 - val_loss: 0.5083 - val_accuracy: 0.9111\n",
      "Learning rate:  1e-05\n",
      "Epoch 149/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1554 - accuracy: 0.9950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1555 - accuracy: 0.9950 - val_loss: 0.5030 - val_accuracy: 0.9136\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  1e-05\n",
      "Epoch 150/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1559 - accuracy: 0.9947WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1559 - accuracy: 0.9947 - val_loss: 0.5030 - val_accuracy: 0.9127\n",
      "Learning rate:  1e-05\n",
      "Epoch 151/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1542 - accuracy: 0.9954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1541 - accuracy: 0.9954 - val_loss: 0.5042 - val_accuracy: 0.9135\n",
      "Learning rate:  1e-05\n",
      "Epoch 152/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1555 - accuracy: 0.9945WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1556 - accuracy: 0.9945 - val_loss: 0.5052 - val_accuracy: 0.9137\n",
      "Learning rate:  1e-05\n",
      "Epoch 153/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1549 - accuracy: 0.9948WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 65ms/step - loss: 0.1549 - accuracy: 0.9948 - val_loss: 0.5054 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-05\n",
      "Epoch 154/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1549 - accuracy: 0.9942WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1550 - accuracy: 0.9942 - val_loss: 0.5074 - val_accuracy: 0.9124\n",
      "Learning rate:  1e-05\n",
      "Epoch 155/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1538 - accuracy: 0.9952WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1538 - accuracy: 0.9952 - val_loss: 0.5064 - val_accuracy: 0.9129\n",
      "Learning rate:  1e-05\n",
      "Epoch 156/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1542 - accuracy: 0.9950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1542 - accuracy: 0.9950 - val_loss: 0.5074 - val_accuracy: 0.9132\n",
      "Learning rate:  1e-05\n",
      "Epoch 157/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1526 - accuracy: 0.9955WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1525 - accuracy: 0.9955 - val_loss: 0.5052 - val_accuracy: 0.9135\n",
      "Learning rate:  1e-05\n",
      "Epoch 158/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1529 - accuracy: 0.9951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1529 - accuracy: 0.9951 - val_loss: 0.5054 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-05\n",
      "Epoch 159/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1536 - accuracy: 0.9950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1537 - accuracy: 0.9950 - val_loss: 0.5078 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-05\n",
      "Epoch 160/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1525 - accuracy: 0.9954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1525 - accuracy: 0.9954 - val_loss: 0.5074 - val_accuracy: 0.9138\n",
      "Learning rate:  1e-05\n",
      "Epoch 161/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1520 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1520 - accuracy: 0.9958 - val_loss: 0.5094 - val_accuracy: 0.9140\n",
      "Learning rate:  1e-06\n",
      "Epoch 162/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1528 - accuracy: 0.9952WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1528 - accuracy: 0.9952 - val_loss: 0.5094 - val_accuracy: 0.9135\n",
      "Learning rate:  1e-06\n",
      "Epoch 163/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1515 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 55ms/step - loss: 0.1514 - accuracy: 0.9956 - val_loss: 0.5095 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 164/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1519 - accuracy: 0.9957WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.1519 - accuracy: 0.9957 - val_loss: 0.5085 - val_accuracy: 0.9128\n",
      "Learning rate:  1e-06\n",
      "Epoch 165/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1521 - accuracy: 0.9953WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1522 - accuracy: 0.9953 - val_loss: 0.5087 - val_accuracy: 0.9128\n",
      "Learning rate:  1e-06\n",
      "Epoch 166/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1522 - accuracy: 0.9954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 56ms/step - loss: 0.1522 - accuracy: 0.9954 - val_loss: 0.5086 - val_accuracy: 0.9136\n",
      "Learning rate:  1e-06\n",
      "Epoch 167/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1514 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 25s 63ms/step - loss: 0.1514 - accuracy: 0.9956 - val_loss: 0.5085 - val_accuracy: 0.9135\n",
      "Learning rate:  1e-06\n",
      "Epoch 168/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1515 - accuracy: 0.9957WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1515 - accuracy: 0.9957 - val_loss: 0.5078 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 169/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1515 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1515 - accuracy: 0.9958 - val_loss: 0.5091 - val_accuracy: 0.9134\n",
      "Learning rate:  1e-06\n",
      "Epoch 170/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1513 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1513 - accuracy: 0.9956 - val_loss: 0.5096 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 171/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1521 - accuracy: 0.9953WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1521 - accuracy: 0.9953 - val_loss: 0.5084 - val_accuracy: 0.9134\n",
      "Learning rate:  1e-06\n",
      "Epoch 172/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1510 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1510 - accuracy: 0.9958 - val_loss: 0.5085 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 173/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1514 - accuracy: 0.9952WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1515 - accuracy: 0.9952 - val_loss: 0.5091 - val_accuracy: 0.9129\n",
      "Learning rate:  1e-06\n",
      "Epoch 174/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1514 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1514 - accuracy: 0.9958 - val_loss: 0.5094 - val_accuracy: 0.9133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  1e-06\n",
      "Epoch 175/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1518 - accuracy: 0.9953WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1517 - accuracy: 0.9953 - val_loss: 0.5098 - val_accuracy: 0.9124\n",
      "Learning rate:  1e-06\n",
      "Epoch 176/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1513 - accuracy: 0.9954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 55ms/step - loss: 0.1513 - accuracy: 0.9954 - val_loss: 0.5094 - val_accuracy: 0.9128\n",
      "Learning rate:  1e-06\n",
      "Epoch 177/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1518 - accuracy: 0.9950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1518 - accuracy: 0.9950 - val_loss: 0.5084 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 178/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1518 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1518 - accuracy: 0.9955 - val_loss: 0.5088 - val_accuracy: 0.9132\n",
      "Learning rate:  1e-06\n",
      "Epoch 179/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1514 - accuracy: 0.9957WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1514 - accuracy: 0.9957 - val_loss: 0.5091 - val_accuracy: 0.9129\n",
      "Learning rate:  1e-06\n",
      "Epoch 180/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1513 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 57ms/step - loss: 0.1513 - accuracy: 0.9956 - val_loss: 0.5082 - val_accuracy: 0.9131\n",
      "Learning rate:  1e-06\n",
      "Epoch 181/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1509 - accuracy: 0.9959WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1510 - accuracy: 0.9958 - val_loss: 0.5079 - val_accuracy: 0.9133\n",
      "Learning rate:  5e-07\n",
      "Epoch 182/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1507 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1508 - accuracy: 0.9958 - val_loss: 0.5085 - val_accuracy: 0.9131\n",
      "Learning rate:  5e-07\n",
      "Epoch 183/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1509 - accuracy: 0.9958WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1509 - accuracy: 0.9958 - val_loss: 0.5085 - val_accuracy: 0.9132\n",
      "Learning rate:  5e-07\n",
      "Epoch 184/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1520 - accuracy: 0.9951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1520 - accuracy: 0.9952 - val_loss: 0.5083 - val_accuracy: 0.9133\n",
      "Learning rate:  5e-07\n",
      "Epoch 185/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1511 - accuracy: 0.9959WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1512 - accuracy: 0.9959 - val_loss: 0.5088 - val_accuracy: 0.9126\n",
      "Learning rate:  5e-07\n",
      "Epoch 186/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1511 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1512 - accuracy: 0.9956 - val_loss: 0.5082 - val_accuracy: 0.9130\n",
      "Learning rate:  5e-07\n",
      "Epoch 187/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1520 - accuracy: 0.9952WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1520 - accuracy: 0.9952 - val_loss: 0.5084 - val_accuracy: 0.9129\n",
      "Learning rate:  5e-07\n",
      "Epoch 188/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1515 - accuracy: 0.9955WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 20s 52ms/step - loss: 0.1515 - accuracy: 0.9956 - val_loss: 0.5080 - val_accuracy: 0.9133\n",
      "Learning rate:  5e-07\n",
      "Epoch 189/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1514 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 53ms/step - loss: 0.1514 - accuracy: 0.9956 - val_loss: 0.5082 - val_accuracy: 0.9131\n",
      "Learning rate:  5e-07\n",
      "Epoch 190/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1512 - accuracy: 0.9959WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 54ms/step - loss: 0.1511 - accuracy: 0.9959 - val_loss: 0.5086 - val_accuracy: 0.9129\n",
      "Learning rate:  5e-07\n",
      "Epoch 191/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1515 - accuracy: 0.9955WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1515 - accuracy: 0.9955 - val_loss: 0.5087 - val_accuracy: 0.9130\n",
      "Learning rate:  5e-07\n",
      "Epoch 192/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1519 - accuracy: 0.9953WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 21s 55ms/step - loss: 0.1519 - accuracy: 0.9953 - val_loss: 0.5083 - val_accuracy: 0.9129\n",
      "Learning rate:  5e-07\n",
      "Epoch 193/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1505 - accuracy: 0.9959WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 22s 55ms/step - loss: 0.1506 - accuracy: 0.9959 - val_loss: 0.5078 - val_accuracy: 0.9129\n",
      "Learning rate:  5e-07\n",
      "Epoch 194/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1521 - accuracy: 0.9948WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 23s 58ms/step - loss: 0.1521 - accuracy: 0.9948 - val_loss: 0.5075 - val_accuracy: 0.9130\n",
      "Learning rate:  5e-07\n",
      "Epoch 195/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1510 - accuracy: 0.9957WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1509 - accuracy: 0.9957 - val_loss: 0.5082 - val_accuracy: 0.9130\n",
      "Learning rate:  5e-07\n",
      "Epoch 196/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1513 - accuracy: 0.9956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1514 - accuracy: 0.9956 - val_loss: 0.5082 - val_accuracy: 0.9134\n",
      "Learning rate:  5e-07\n",
      "Epoch 197/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1519 - accuracy: 0.9950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1519 - accuracy: 0.9950 - val_loss: 0.5082 - val_accuracy: 0.9127\n",
      "Learning rate:  5e-07\n",
      "Epoch 198/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1516 - accuracy: 0.9951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1516 - accuracy: 0.9951 - val_loss: 0.5071 - val_accuracy: 0.9131\n",
      "Learning rate:  5e-07\n",
      "Epoch 199/200\n",
      "389/391 [============================>.] - ETA: 0s - loss: 0.1508 - accuracy: 0.9954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 67ms/step - loss: 0.1508 - accuracy: 0.9955 - val_loss: 0.5075 - val_accuracy: 0.9128\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  5e-07\n",
      "Epoch 200/200\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.1511 - accuracy: 0.9955WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 26s 66ms/step - loss: 0.1512 - accuracy: 0.9954 - val_loss: 0.5075 - val_accuracy: 0.9138\n"
     ]
    }
   ],
   "source": [
    "# Training parameters\n",
    "batch_size = 128  # orig paper trained all networks with batch_size=128\n",
    "epochs = 100\n",
    "data_augmentation = True\n",
    "num_classes = 10\n",
    "\n",
    "# Subtracting pixel mean improves accuracy\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "n = 5\n",
    "\n",
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 2\n",
    "\n",
    "#Computed depth from supplied model parameter n\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "\n",
    "# Model name, depth and version\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "\n",
    "# Load the CIFAR10 data.\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "print('y_train shape:', y_train.shape)\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth)\n",
    "\n",
    "model = tfmot.quantization.keras.quantize_model(model)\n",
    "    \n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(learning_rate=lr_schedule(0)),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "print(model_type)\n",
    "\n",
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]\n",
    "\n",
    "# Run training, with or without data augmentation.\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=callbacks)\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs, verbose=1, workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('Resnet18-200epochs.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 213us/sample - loss: 0.5075 - accuracy: 0.9138\n",
      "Test loss: 0.5075194211006164\n",
      "Test accuracy: 0.9138\n"
     ]
    }
   ],
   "source": [
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quantization Aware Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "y_train shape: (50000, 1)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e0034e590>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e0034e590>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e00356e50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e00356e50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e00226ad0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e00226ad0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001be3d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001be3d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ca990>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ca990>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7f8e00343ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001e1910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001e1910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001e1ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001e1ed0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ed4d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ed4d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ed910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7f8e001ed910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "Learning rate:  0.001\n",
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "quantize_layer_2 (QuantizeLayer (None, 32, 32, 3)    3           input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_44 (QuantizeWrappe (None, 32, 32, 16)   483         quantize_layer_2[1][0]           \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_38 (QuantizeWr (None, 32, 32, 16)   3           quant_conv2d_44[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_45 (QuantizeWrappe (None, 32, 32, 16)   307         quant_activation_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_39 (QuantizeWr (None, 32, 32, 16)   3           quant_conv2d_45[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_46 (QuantizeWrappe (None, 32, 32, 16)   2355        quant_activation_39[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_40 (QuantizeWr (None, 32, 32, 16)   3           quant_conv2d_46[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_48 (QuantizeWrappe (None, 32, 32, 64)   1219        quant_activation_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_47 (QuantizeWrappe (None, 32, 32, 64)   1219        quant_activation_40[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_12 (QuantizeWrapper)  (None, 32, 32, 64)   3           quant_conv2d_48[0][0]            \n",
      "                                                                 quant_conv2d_47[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_41 (QuantizeWr (None, 32, 32, 64)   3           quant_add_12[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_49 (QuantizeWrappe (None, 32, 32, 16)   1075        quant_activation_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_42 (QuantizeWr (None, 32, 32, 16)   3           quant_conv2d_49[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_50 (QuantizeWrappe (None, 32, 32, 16)   2355        quant_activation_42[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_43 (QuantizeWr (None, 32, 32, 16)   3           quant_conv2d_50[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_51 (QuantizeWrappe (None, 32, 32, 64)   1219        quant_activation_43[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_13 (QuantizeWrapper)  (None, 32, 32, 64)   3           quant_add_12[0][0]               \n",
      "                                                                 quant_conv2d_51[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_44 (QuantizeWr (None, 32, 32, 64)   3           quant_add_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_52 (QuantizeWrappe (None, 16, 16, 64)   4291        quant_activation_44[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_45 (QuantizeWr (None, 16, 16, 64)   3           quant_conv2d_52[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_53 (QuantizeWrappe (None, 16, 16, 64)   37059       quant_activation_45[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_46 (QuantizeWr (None, 16, 16, 64)   3           quant_conv2d_53[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_55 (QuantizeWrappe (None, 16, 16, 128)  8579        quant_add_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_54 (QuantizeWrappe (None, 16, 16, 128)  8579        quant_activation_46[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_14 (QuantizeWrapper)  (None, 16, 16, 128)  3           quant_conv2d_55[0][0]            \n",
      "                                                                 quant_conv2d_54[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_47 (QuantizeWr (None, 16, 16, 128)  3           quant_add_14[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_56 (QuantizeWrappe (None, 16, 16, 64)   8387        quant_activation_47[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_48 (QuantizeWr (None, 16, 16, 64)   3           quant_conv2d_56[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_57 (QuantizeWrappe (None, 16, 16, 64)   37059       quant_activation_48[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_49 (QuantizeWr (None, 16, 16, 64)   3           quant_conv2d_57[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_58 (QuantizeWrappe (None, 16, 16, 128)  8579        quant_activation_49[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_15 (QuantizeWrapper)  (None, 16, 16, 128)  3           quant_add_14[0][0]               \n",
      "                                                                 quant_conv2d_58[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_50 (QuantizeWr (None, 16, 16, 128)  3           quant_add_15[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_59 (QuantizeWrappe (None, 8, 8, 128)    16771       quant_activation_50[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_51 (QuantizeWr (None, 8, 8, 128)    3           quant_conv2d_59[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_60 (QuantizeWrappe (None, 8, 8, 128)    147843      quant_activation_51[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_52 (QuantizeWr (None, 8, 8, 128)    3           quant_conv2d_60[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_62 (QuantizeWrappe (None, 8, 8, 256)    33539       quant_add_15[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_61 (QuantizeWrappe (None, 8, 8, 256)    33539       quant_activation_52[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_16 (QuantizeWrapper)  (None, 8, 8, 256)    3           quant_conv2d_62[0][0]            \n",
      "                                                                 quant_conv2d_61[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_53 (QuantizeWr (None, 8, 8, 256)    3           quant_add_16[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_63 (QuantizeWrappe (None, 8, 8, 128)    33155       quant_activation_53[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_54 (QuantizeWr (None, 8, 8, 128)    3           quant_conv2d_63[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_64 (QuantizeWrappe (None, 8, 8, 128)    147843      quant_activation_54[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_55 (QuantizeWr (None, 8, 8, 128)    3           quant_conv2d_64[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_conv2d_65 (QuantizeWrappe (None, 8, 8, 256)    33539       quant_activation_55[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_add_17 (QuantizeWrapper)  (None, 8, 8, 256)    3           quant_add_16[0][0]               \n",
      "                                                                 quant_conv2d_65[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "quant_activation_56 (QuantizeWr (None, 8, 8, 256)    3           quant_add_17[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "quant_average_pooling2d_2 (Quan (None, 1, 1, 256)    3           quant_activation_56[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "quant_flatten_2 (QuantizeWrappe (None, 256)          1           quant_average_pooling2d_2[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "quant_dense_2 (QuantizeWrapper) (None, 10)           2575        quant_flatten_2[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 571,651\n",
      "Trainable params: 567,114\n",
      "Non-trainable params: 4,537\n",
      "__________________________________________________________________________________________________\n",
      "ResNet20v2\n",
      "Using real-time data augmentation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-13-c65730a9ba72>:149: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use Model.fit, which supports generators.\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train for 391 steps, validate on 10000 samples\n",
      "Learning rate:  0.001\n",
      "Epoch 1/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 2.1348 - accuracy: 0.3532WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 62s 160ms/step - loss: 2.1336 - accuracy: 0.3536 - val_loss: 1.9176 - val_accuracy: 0.4181\n",
      "Learning rate:  0.001\n",
      "Epoch 2/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.7430 - accuracy: 0.4874WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 1.7424 - accuracy: 0.4876 - val_loss: 1.6491 - val_accuracy: 0.5116\n",
      "Learning rate:  0.001\n",
      "Epoch 3/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.5523 - accuracy: 0.5550WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 47s 120ms/step - loss: 1.5520 - accuracy: 0.5551 - val_loss: 1.5222 - val_accuracy: 0.5598\n",
      "Learning rate:  0.001\n",
      "Epoch 4/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.4021 - accuracy: 0.6050WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 1.4016 - accuracy: 0.6052 - val_loss: 1.3031 - val_accuracy: 0.6420\n",
      "Learning rate:  0.001\n",
      "Epoch 5/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.2995 - accuracy: 0.6425WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 1.2994 - accuracy: 0.6425 - val_loss: 1.2483 - val_accuracy: 0.6554\n",
      "Learning rate:  0.001\n",
      "Epoch 6/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.2088 - accuracy: 0.6726WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 1.2086 - accuracy: 0.6727 - val_loss: 1.2343 - val_accuracy: 0.6665\n",
      "Learning rate:  0.001\n",
      "Epoch 7/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.1368 - accuracy: 0.6980WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 1.1367 - accuracy: 0.6981 - val_loss: 1.1224 - val_accuracy: 0.7072\n",
      "Learning rate:  0.001\n",
      "Epoch 8/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.0807 - accuracy: 0.7171WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 1.0809 - accuracy: 0.7169 - val_loss: 1.0590 - val_accuracy: 0.7237\n",
      "Learning rate:  0.001\n",
      "Epoch 9/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 1.0208 - accuracy: 0.7375WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 1.0209 - accuracy: 0.7376 - val_loss: 1.0049 - val_accuracy: 0.7489\n",
      "Learning rate:  0.001\n",
      "Epoch 10/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.9777 - accuracy: 0.7519WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.9779 - accuracy: 0.7518 - val_loss: 0.9243 - val_accuracy: 0.7736\n",
      "Learning rate:  0.001\n",
      "Epoch 11/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.9413 - accuracy: 0.7623WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.9412 - accuracy: 0.7623 - val_loss: 0.9906 - val_accuracy: 0.7540\n",
      "Learning rate:  0.001\n",
      "Epoch 12/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.9147 - accuracy: 0.7717WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.9148 - accuracy: 0.7717 - val_loss: 0.9648 - val_accuracy: 0.7677\n",
      "Learning rate:  0.001\n",
      "Epoch 13/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8867 - accuracy: 0.7803WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.8864 - accuracy: 0.7803 - val_loss: 0.9483 - val_accuracy: 0.7685\n",
      "Learning rate:  0.001\n",
      "Epoch 14/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8563 - accuracy: 0.7907WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 0.8561 - accuracy: 0.7908 - val_loss: 0.8941 - val_accuracy: 0.7833\n",
      "Learning rate:  0.001\n",
      "Epoch 15/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8360 - accuracy: 0.7966WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.8363 - accuracy: 0.7965 - val_loss: 0.8588 - val_accuracy: 0.7903\n",
      "Learning rate:  0.001\n",
      "Epoch 16/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8198 - accuracy: 0.8021WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.8199 - accuracy: 0.8020 - val_loss: 0.8428 - val_accuracy: 0.8002\n",
      "Learning rate:  0.001\n",
      "Epoch 17/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.8002 - accuracy: 0.8065WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 0.8006 - accuracy: 0.8064 - val_loss: 0.8239 - val_accuracy: 0.8078\n",
      "Learning rate:  0.001\n",
      "Epoch 18/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7815 - accuracy: 0.8137WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.7812 - accuracy: 0.8138 - val_loss: 0.8067 - val_accuracy: 0.8111\n",
      "Learning rate:  0.001\n",
      "Epoch 19/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7641 - accuracy: 0.8205WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.7644 - accuracy: 0.8203 - val_loss: 0.8297 - val_accuracy: 0.7996\n",
      "Learning rate:  0.001\n",
      "Epoch 20/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7511 - accuracy: 0.8237WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 124ms/step - loss: 0.7513 - accuracy: 0.8237 - val_loss: 0.8335 - val_accuracy: 0.8051\n",
      "Learning rate:  0.001\n",
      "Epoch 21/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7434 - accuracy: 0.8252WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.7437 - accuracy: 0.8251 - val_loss: 0.7915 - val_accuracy: 0.8151\n",
      "Learning rate:  0.001\n",
      "Epoch 22/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7230 - accuracy: 0.8327WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 124ms/step - loss: 0.7230 - accuracy: 0.8327 - val_loss: 0.8126 - val_accuracy: 0.8099\n",
      "Learning rate:  0.001\n",
      "Epoch 23/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7129 - accuracy: 0.8329WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.7131 - accuracy: 0.8328 - val_loss: 0.7713 - val_accuracy: 0.8196\n",
      "Learning rate:  0.001\n",
      "Epoch 24/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.7013 - accuracy: 0.8393WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 127ms/step - loss: 0.7012 - accuracy: 0.8394 - val_loss: 0.9057 - val_accuracy: 0.7896\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 25/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6961 - accuracy: 0.8402WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.6956 - accuracy: 0.8404 - val_loss: 0.7508 - val_accuracy: 0.8198\n",
      "Learning rate:  0.001\n",
      "Epoch 26/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6919 - accuracy: 0.8421WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.6922 - accuracy: 0.8420 - val_loss: 0.8035 - val_accuracy: 0.8153\n",
      "Learning rate:  0.001\n",
      "Epoch 27/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6721 - accuracy: 0.8479WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.6721 - accuracy: 0.8479 - val_loss: 0.7856 - val_accuracy: 0.8154\n",
      "Learning rate:  0.001\n",
      "Epoch 28/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6645 - accuracy: 0.8507WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 124ms/step - loss: 0.6641 - accuracy: 0.8508 - val_loss: 0.7534 - val_accuracy: 0.8273\n",
      "Learning rate:  0.001\n",
      "Epoch 29/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6652 - accuracy: 0.8505WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.6654 - accuracy: 0.8504 - val_loss: 0.7608 - val_accuracy: 0.8276\n",
      "Learning rate:  0.001\n",
      "Epoch 30/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6493 - accuracy: 0.8559WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.6492 - accuracy: 0.8559 - val_loss: 0.7719 - val_accuracy: 0.8221\n",
      "Learning rate:  0.001\n",
      "Epoch 31/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6406 - accuracy: 0.8579WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 127ms/step - loss: 0.6406 - accuracy: 0.8580 - val_loss: 0.7751 - val_accuracy: 0.8230\n",
      "Learning rate:  0.001\n",
      "Epoch 32/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6304 - accuracy: 0.8619WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.6302 - accuracy: 0.8621 - val_loss: 0.7652 - val_accuracy: 0.8256\n",
      "Learning rate:  0.001\n",
      "Epoch 33/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6269 - accuracy: 0.8636WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.6269 - accuracy: 0.8636 - val_loss: 0.7080 - val_accuracy: 0.8365\n",
      "Learning rate:  0.001\n",
      "Epoch 34/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6151 - accuracy: 0.8656WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.6151 - accuracy: 0.8656 - val_loss: 0.6940 - val_accuracy: 0.8437\n",
      "Learning rate:  0.001\n",
      "Epoch 35/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6125 - accuracy: 0.8670WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.6124 - accuracy: 0.8670 - val_loss: 0.7712 - val_accuracy: 0.8236\n",
      "Learning rate:  0.001\n",
      "Epoch 36/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6071 - accuracy: 0.8691WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.6070 - accuracy: 0.8691 - val_loss: 0.7390 - val_accuracy: 0.8344\n",
      "Learning rate:  0.001\n",
      "Epoch 37/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.6047 - accuracy: 0.8705WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.6046 - accuracy: 0.8704 - val_loss: 0.7082 - val_accuracy: 0.8429\n",
      "Learning rate:  0.001\n",
      "Epoch 38/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5941 - accuracy: 0.8745WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5947 - accuracy: 0.8743 - val_loss: 0.7584 - val_accuracy: 0.8230\n",
      "Learning rate:  0.001\n",
      "Epoch 39/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5955 - accuracy: 0.8729WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5953 - accuracy: 0.8729 - val_loss: 0.7125 - val_accuracy: 0.8405\n",
      "Learning rate:  0.001\n",
      "Epoch 40/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5825 - accuracy: 0.8778WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 122ms/step - loss: 0.5824 - accuracy: 0.8778 - val_loss: 0.7183 - val_accuracy: 0.8439\n",
      "Learning rate:  0.001\n",
      "Epoch 41/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5825 - accuracy: 0.8772WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.5826 - accuracy: 0.8772 - val_loss: 0.7549 - val_accuracy: 0.8348\n",
      "Learning rate:  0.001\n",
      "Epoch 42/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5698 - accuracy: 0.8810WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.5701 - accuracy: 0.8808 - val_loss: 0.7244 - val_accuracy: 0.8443\n",
      "Learning rate:  0.001\n",
      "Epoch 43/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5672 - accuracy: 0.8832WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 124ms/step - loss: 0.5676 - accuracy: 0.8831 - val_loss: 0.6518 - val_accuracy: 0.8584\n",
      "Learning rate:  0.001\n",
      "Epoch 44/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5721 - accuracy: 0.8806WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.5718 - accuracy: 0.8806 - val_loss: 0.7306 - val_accuracy: 0.8407\n",
      "Learning rate:  0.001\n",
      "Epoch 45/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5625 - accuracy: 0.8832WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 125ms/step - loss: 0.5624 - accuracy: 0.8832 - val_loss: 0.7629 - val_accuracy: 0.8275\n",
      "Learning rate:  0.001\n",
      "Epoch 46/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5589 - accuracy: 0.8843WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.5590 - accuracy: 0.8843 - val_loss: 0.6797 - val_accuracy: 0.8586\n",
      "Learning rate:  0.001\n",
      "Epoch 47/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5521 - accuracy: 0.8876WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5520 - accuracy: 0.8876 - val_loss: 0.7139 - val_accuracy: 0.8448\n",
      "Learning rate:  0.001\n",
      "Epoch 48/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5445 - accuracy: 0.8894WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5445 - accuracy: 0.8895 - val_loss: 0.6532 - val_accuracy: 0.8602\n",
      "Learning rate:  0.001\n",
      "Epoch 49/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5440 - accuracy: 0.8895WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 122ms/step - loss: 0.5437 - accuracy: 0.8897 - val_loss: 0.6904 - val_accuracy: 0.8514\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 50/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5394 - accuracy: 0.8906WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.5394 - accuracy: 0.8906 - val_loss: 0.6797 - val_accuracy: 0.8541\n",
      "Learning rate:  0.001\n",
      "Epoch 51/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5326 - accuracy: 0.8937WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.5329 - accuracy: 0.8937 - val_loss: 0.7073 - val_accuracy: 0.8458\n",
      "Learning rate:  0.001\n",
      "Epoch 52/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5361 - accuracy: 0.8915WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5360 - accuracy: 0.8917 - val_loss: 0.6956 - val_accuracy: 0.8519\n",
      "Learning rate:  0.001\n",
      "Epoch 53/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5314 - accuracy: 0.8956WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 122ms/step - loss: 0.5312 - accuracy: 0.8956 - val_loss: 0.7290 - val_accuracy: 0.8448\n",
      "Learning rate:  0.001\n",
      "Epoch 54/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5255 - accuracy: 0.8962WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 123ms/step - loss: 0.5254 - accuracy: 0.8962 - val_loss: 0.7040 - val_accuracy: 0.8519\n",
      "Learning rate:  0.001\n",
      "Epoch 55/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5254 - accuracy: 0.8941WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 127ms/step - loss: 0.5252 - accuracy: 0.8942 - val_loss: 0.6691 - val_accuracy: 0.8605\n",
      "Learning rate:  0.001\n",
      "Epoch 56/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5276 - accuracy: 0.8951WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5276 - accuracy: 0.8950 - val_loss: 0.6970 - val_accuracy: 0.8476\n",
      "Learning rate:  0.001\n",
      "Epoch 57/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5189 - accuracy: 0.8987WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 128ms/step - loss: 0.5192 - accuracy: 0.8986 - val_loss: 0.6690 - val_accuracy: 0.8589\n",
      "Learning rate:  0.001\n",
      "Epoch 58/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5097 - accuracy: 0.9017WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5097 - accuracy: 0.9016 - val_loss: 0.6922 - val_accuracy: 0.8520\n",
      "Learning rate:  0.001\n",
      "Epoch 59/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5175 - accuracy: 0.8972WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 0.5175 - accuracy: 0.8972 - val_loss: 0.6878 - val_accuracy: 0.8532\n",
      "Learning rate:  0.001\n",
      "Epoch 60/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5077 - accuracy: 0.9010WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 126ms/step - loss: 0.5076 - accuracy: 0.9010 - val_loss: 0.7073 - val_accuracy: 0.8520\n",
      "Learning rate:  0.001\n",
      "Epoch 61/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5037 - accuracy: 0.9028WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.5038 - accuracy: 0.9027 - val_loss: 0.6648 - val_accuracy: 0.8644\n",
      "Learning rate:  0.001\n",
      "Epoch 62/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5040 - accuracy: 0.9038WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.5043 - accuracy: 0.9037 - val_loss: 0.7286 - val_accuracy: 0.8443\n",
      "Learning rate:  0.001\n",
      "Epoch 63/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.5031 - accuracy: 0.9025WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.5032 - accuracy: 0.9024 - val_loss: 0.6894 - val_accuracy: 0.8566\n",
      "Learning rate:  0.001\n",
      "Epoch 64/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4973 - accuracy: 0.9044WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4973 - accuracy: 0.9044 - val_loss: 0.6942 - val_accuracy: 0.8578\n",
      "Learning rate:  0.001\n",
      "Epoch 65/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4990 - accuracy: 0.9039WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4991 - accuracy: 0.9038 - val_loss: 0.6656 - val_accuracy: 0.8655\n",
      "Learning rate:  0.001\n",
      "Epoch 66/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4963 - accuracy: 0.9050WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4963 - accuracy: 0.9050 - val_loss: 0.6921 - val_accuracy: 0.8554\n",
      "Learning rate:  0.001\n",
      "Epoch 67/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4919 - accuracy: 0.9064WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4921 - accuracy: 0.9064 - val_loss: 0.7784 - val_accuracy: 0.8359\n",
      "Learning rate:  0.001\n",
      "Epoch 68/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4907 - accuracy: 0.9072WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4906 - accuracy: 0.9073 - val_loss: 0.7070 - val_accuracy: 0.8489\n",
      "Learning rate:  0.001\n",
      "Epoch 69/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4867 - accuracy: 0.9091WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4869 - accuracy: 0.9091 - val_loss: 0.7383 - val_accuracy: 0.8480\n",
      "Learning rate:  0.001\n",
      "Epoch 70/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4888 - accuracy: 0.9080WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 132ms/step - loss: 0.4890 - accuracy: 0.9079 - val_loss: 0.6459 - val_accuracy: 0.8656\n",
      "Learning rate:  0.001\n",
      "Epoch 71/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4947 - accuracy: 0.9062WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.4949 - accuracy: 0.9060 - val_loss: 0.7151 - val_accuracy: 0.8515\n",
      "Learning rate:  0.001\n",
      "Epoch 72/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4834 - accuracy: 0.9106WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.4838 - accuracy: 0.9104 - val_loss: 0.7245 - val_accuracy: 0.8472\n",
      "Learning rate:  0.001\n",
      "Epoch 73/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4859 - accuracy: 0.9088WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.4858 - accuracy: 0.9088 - val_loss: 0.7048 - val_accuracy: 0.8546\n",
      "Learning rate:  0.001\n",
      "Epoch 74/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4745 - accuracy: 0.9137WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.4745 - accuracy: 0.9136 - val_loss: 0.6941 - val_accuracy: 0.8549\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Epoch 75/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4811 - accuracy: 0.9097WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.4810 - accuracy: 0.9097 - val_loss: 0.6964 - val_accuracy: 0.8589\n",
      "Learning rate:  0.001\n",
      "Epoch 76/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4770 - accuracy: 0.9113WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.4769 - accuracy: 0.9113 - val_loss: 0.6724 - val_accuracy: 0.8642\n",
      "Learning rate:  0.001\n",
      "Epoch 77/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4742 - accuracy: 0.9127WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 48s 122ms/step - loss: 0.4745 - accuracy: 0.9126 - val_loss: 0.6913 - val_accuracy: 0.8554\n",
      "Learning rate:  0.001\n",
      "Epoch 78/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4709 - accuracy: 0.9144WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 49s 124ms/step - loss: 0.4712 - accuracy: 0.9143 - val_loss: 0.8094 - val_accuracy: 0.8388\n",
      "Learning rate:  0.001\n",
      "Epoch 79/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4726 - accuracy: 0.9115WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.4727 - accuracy: 0.9115 - val_loss: 0.7185 - val_accuracy: 0.8531\n",
      "Learning rate:  0.001\n",
      "Epoch 80/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4687 - accuracy: 0.9149WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.4687 - accuracy: 0.9148 - val_loss: 0.6609 - val_accuracy: 0.8694\n",
      "Learning rate:  0.001\n",
      "Epoch 81/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.4729 - accuracy: 0.9125WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 52s 133ms/step - loss: 0.4728 - accuracy: 0.9125 - val_loss: 0.6843 - val_accuracy: 0.8590\n",
      "Learning rate:  0.0001\n",
      "Epoch 82/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3755 - accuracy: 0.9461WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3754 - accuracy: 0.9461 - val_loss: 0.6322 - val_accuracy: 0.8803\n",
      "Learning rate:  0.0001\n",
      "Epoch 83/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3537 - accuracy: 0.9547WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.3537 - accuracy: 0.9547 - val_loss: 0.6198 - val_accuracy: 0.8847\n",
      "Learning rate:  0.0001\n",
      "Epoch 84/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3472 - accuracy: 0.9576WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.3471 - accuracy: 0.9576 - val_loss: 0.6371 - val_accuracy: 0.8837\n",
      "Learning rate:  0.0001\n",
      "Epoch 85/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3402 - accuracy: 0.9592WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3402 - accuracy: 0.9592 - val_loss: 0.6318 - val_accuracy: 0.8840\n",
      "Learning rate:  0.0001\n",
      "Epoch 86/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3317 - accuracy: 0.9621WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3315 - accuracy: 0.9622 - val_loss: 0.6353 - val_accuracy: 0.8883\n",
      "Learning rate:  0.0001\n",
      "Epoch 87/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3288 - accuracy: 0.9625WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3288 - accuracy: 0.9625 - val_loss: 0.6630 - val_accuracy: 0.8857\n",
      "Learning rate:  0.0001\n",
      "Epoch 88/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3219 - accuracy: 0.9641WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3218 - accuracy: 0.9641 - val_loss: 0.6461 - val_accuracy: 0.8853\n",
      "Learning rate:  0.0001\n",
      "Epoch 89/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3194 - accuracy: 0.9648WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3195 - accuracy: 0.9648 - val_loss: 0.6356 - val_accuracy: 0.8883\n",
      "Learning rate:  0.0001\n",
      "Epoch 90/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3180 - accuracy: 0.9640WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3178 - accuracy: 0.9641 - val_loss: 0.6570 - val_accuracy: 0.8850\n",
      "Learning rate:  0.0001\n",
      "Epoch 91/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3115 - accuracy: 0.9669WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3115 - accuracy: 0.9670 - val_loss: 0.6466 - val_accuracy: 0.8876\n",
      "Learning rate:  0.0001\n",
      "Epoch 92/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3095 - accuracy: 0.9670WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3094 - accuracy: 0.9670 - val_loss: 0.6462 - val_accuracy: 0.8904\n",
      "Learning rate:  0.0001\n",
      "Epoch 93/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3044 - accuracy: 0.9686WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 131ms/step - loss: 0.3044 - accuracy: 0.9686 - val_loss: 0.6428 - val_accuracy: 0.8914\n",
      "Learning rate:  0.0001\n",
      "Epoch 94/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.3039 - accuracy: 0.9682WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.3039 - accuracy: 0.9682 - val_loss: 0.6635 - val_accuracy: 0.8856\n",
      "Learning rate:  0.0001\n",
      "Epoch 95/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2994 - accuracy: 0.9700WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 129ms/step - loss: 0.2994 - accuracy: 0.9700 - val_loss: 0.6705 - val_accuracy: 0.8881\n",
      "Learning rate:  0.0001\n",
      "Epoch 96/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2970 - accuracy: 0.9694WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 129ms/step - loss: 0.2969 - accuracy: 0.9695 - val_loss: 0.6582 - val_accuracy: 0.8866\n",
      "Learning rate:  0.0001\n",
      "Epoch 97/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2959 - accuracy: 0.9702WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 130ms/step - loss: 0.2957 - accuracy: 0.9703 - val_loss: 0.6691 - val_accuracy: 0.8907\n",
      "Learning rate:  0.0001\n",
      "Epoch 98/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2924 - accuracy: 0.9703WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 129ms/step - loss: 0.2925 - accuracy: 0.9703 - val_loss: 0.6897 - val_accuracy: 0.8863\n",
      "Learning rate:  0.0001\n",
      "Epoch 99/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2909 - accuracy: 0.9719WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 51s 129ms/step - loss: 0.2909 - accuracy: 0.9719 - val_loss: 0.6736 - val_accuracy: 0.8870\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.0001\n",
      "Epoch 100/100\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.2879 - accuracy: 0.9719WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "391/391 [==============================] - 50s 129ms/step - loss: 0.2879 - accuracy: 0.9719 - val_loss: 0.6658 - val_accuracy: 0.8885\n"
     ]
    }
   ],
   "source": [
    "# Training parameters\n",
    "batch_size = 128  # orig paper trained all networks with batch_size=128\n",
    "epochs = 100\n",
    "data_augmentation = True\n",
    "num_classes = 10\n",
    "\n",
    "# Subtracting pixel mean improves accuracy\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "n = 5\n",
    "\n",
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 2\n",
    "\n",
    "#Computed depth from supplied model parameter n\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "\n",
    "# Model name, depth and version\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "\n",
    "# Load the CIFAR10 data.\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "print('y_train shape:', y_train.shape)\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth)\n",
    "\n",
    "model = tfmot.quantization.keras.quantize_model(model)\n",
    "    \n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(learning_rate=lr_schedule(0)),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "print(model_type)\n",
    "\n",
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]\n",
    "\n",
    "# Run training, with or without data augmentation.\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=callbacks)\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs, verbose=1, workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('QAT-Resnet18.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/rishi/anaconda3/envs/tensorflow1.1/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1635: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe80b90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe80b90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe87910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe87910>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe2bb90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe2bb90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe37d50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe37d50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe40710>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe40710>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afdc90d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afdc90d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc9b10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc9b10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdd6990>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdd6990>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afddd810>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afddd810>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afde81d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afde81d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afde8bd0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afde8bd0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdf3a50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdf3a50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdfd8d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdfd8d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe06290>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afe06290>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afe06c10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afe06c10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd8f650>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd8f650>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd9b4d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd9b4d0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afda2350>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afda2350>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afda2cd0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afda2cd0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdac710>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdac710>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdb6590>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdb6590>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc0410>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc0410>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc0d90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afdc0d90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd4a750>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd4a750>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd54190>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd54190>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd54c50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd54c50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd5de50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitConvQuantizeConfig object at 0x7ff0afd5de50>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd67810>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd67810>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70250>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70250>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70950>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70950>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70f10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n",
      "WARNING: AutoGraph could not transform <bound method Default8BitQuantizeConfig.set_quantize_activations of <tensorflow_model_optimization.python.core.quantization.keras.default_8bit.default_8bit_quantize_registry.Default8BitQuantizeConfig object at 0x7ff0afd70f10>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: expected an indented block (<unknown>, line 14)\n"
     ]
    }
   ],
   "source": [
    "with tfmot.quantization.keras.quantize_scope():\n",
    "    model = tf.keras.models.load_model('QAT-Resnet18.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = KerasClassifier(model=model, clip_values=None, use_logits=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "y_train shape: (50000, 1)\n"
     ]
    }
   ],
   "source": [
    "# Load the CIFAR10 data.\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if True:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "print('y_train shape:', y_train.shape)\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, 10)\n",
    "y_test = keras.utils.to_categorical(y_test, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on fgsm 0.1 test examples: 76.94%\n"
     ]
    }
   ],
   "source": [
    "from numpy import load\n",
    "data = load('fgsmtest(0.01).npy') # attack images of the resnet-200 epoch model, not quantized\n",
    "predictions = model.predict(data)\n",
    "accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on fgsm 0.1 test examples: {}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on fgsm 0.03 test examples: 21.790000000000003%\n"
     ]
    }
   ],
   "source": [
    "fgsm1 = FastGradientMethod(classifier, eps=0.03)\n",
    "test = fgsm1.generate(x_test)\n",
    "predictions = model.predict(test)\n",
    "accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on fgsm 0.03 test examples: {}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from art.defences import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on adversarial test examples: 81.95%\n",
      "Accuracy on adversarial test examples: 14.899999999999999%\n"
     ]
    }
   ],
   "source": [
    "ss = SpatialSmoothing(window_size=3)\n",
    "\n",
    "x_art_def, _ = ss(x_test)\n",
    "x_art_adv_def, _ = ss(x_test_adv)\n",
    "\n",
    "pred1 = classifier.predict(x_art_def)\n",
    "pred2 = classifier.predict(x_art_adv_def)\n",
    "\n",
    "accuracy = np.sum(np.argmax(pred1, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on adversarial test examples: {}%\".format(accuracy * 100))\n",
    "\n",
    "accuracy = np.sum(np.argmax(pred2, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on adversarial test examples: {}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n",
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwAAAAI/CAYAAADA2r5CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdbZBcd5Xn+d/JzHouyXqWhSRbxjYYY8AGhbfZZroNHfQYZmMNER1sO2I6PBHEiBdNBMT0iyF4A92xG8FMNLC8YkIMXkwEjZvgYWE3iJ52MN6lmR5syw/YsgW2bMtYsp6lUlWpHjPz7ItKr2W57vlX5c2sTOn//UQoVJUn//eeupWV/3vyZp6/ubsAAAAA5KHS6wQAAAAArB0KAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMlIrM9jM7pb0DUlVSf/Z3b+SuD89R4GrlLtbr3MAsPq5eXh42MfGx9rbWdf/7EtsP3HK0d0u6Il9d3PXCakjWua4pLbdTI2PNlDyoHmJDTTrceaNZqPtbUtSbaD4dLySOKrRTzU7c1Hz8/PLbsDaXQfAzKqSnpf0UUlHJT0m6V53fy4YQwEAXKUoAIDea2du3rxls3/sf/54W/trLlbbGvfGBuJwxQbaHSpV43s0vZ7aQtuSJ4zJ7IvfoGHVxFNtI952LfHmj0UvPpkNQpKWKs7IQuIssFr86y5dsTUTBUCzXhyfnpgOx16cutBWTq/bvHN7YWzI4tfqo/P4/+e/PqSJ8+eWfcCUeQvQnZIOu/tL7r4g6UFJ95TYHgAAKIe5GUBSmQJgp6RXL/n+aOs2AADQG8zNAJJKfQZgJcxsn6R93d4PAABYmUvn5tGxNt//D+CKVeYKwDFJuy/5flfrtjdx9/3uvtfd95bYFwAASFv13Dw8PLRmyQHoD2UKgMck3WxmN5jZoKQ/l/SzzqQFAADawNwMIKnttwC5e93MPivpv2jpg9/3u/uzHcsMAACsSntzs0mJTiOFI1OtNlNtYVKNeCpB25nExpNdgrqpFnfqqXjq9df2m6pVBxLbTnVeCnr5uMq1u4ya/Ehxao1Ua9XEY2lxcSGMz1+cL4z95rED8cYTbrrl5jBe9eK/Pw97o6rth0qpzwC4+88l/bzMNgAAQOcwNwNIYSVgAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAy0vWVgAEAQB8zSZXi1wMt6M1oJV9G9GqiJ2XQ4rDSTLTatLhNaD3RwrQSxD2RdrUSbzvR0TJstpnqrGqpdpmJDUQdTOvVxODiTpoti2G03ijOfWriYjj2N4/HrTrPnz0Rxrvp0UdOhvH/6ZP/S2HME6/Vx7/t4l8mVwAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICOsAwAAQNZMHnSXD5YIkJqJnvNRI39JjUQX81ql+DSlHo6UUi3rLdFR31Xc7L+Z7PMf/9y1xAIKUTS1BkEjyHtpfByPjls1/rHkid/nbOKX9uRTzxbGjh4+GA++gj366/+3MPY/fPCucKwl1rsowhUAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgI6wAAAJC59jqJS/VUH//EOgCW6IdvUT/9gbiffSXxGmfD49yr4VGJx3bz1dVmNbEGQTNxzMOFHaRGuLZDvO25+lwY/+f/9kQYP3v8+TB+tTp1/ERhLLW2QmJphkJcAQAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjLAOAAAAmWsEzcSjE4XaQGrDca9+JcLx2Pg1zMXkBhJrEAQd1msW92ZPdOpPNm+vt9+KX5ZsDB8fdA8OnHs9HEuf/867OHUhjK/btLk42ObfdZKZHZE0Jakhqe7ue8tsDwAAlMPcDCClE1cAPuzuZzqwHQAA0BnMzQAK8RkAAAAAICNlCwCX9I9m9riZ7etEQgAAoBTmZgChsm8B+pC7HzOzbZIeMrPfuvsvL71D68mHJyAAANbGqubm0fHxXuQIoIdKXQFw92Ot/09J+omkO5e5z35338uHkAAA6L7Vzs3Dw8NrnSKAHmv7CoCZjUmquPtU6+s/lfQ3HcsMCKQq1/WJ+FC1/X3bYBw/Mdv+tgGgjHbmZpOpZsGTYtCSst6IG15WE6cZ1UQbUQ9bWqaabcYzxWAl7pfp4eYTbUCT7U0T44OdW7LPZ+J3Ev2uJTWj38liIxxLm8/Oe/53vwvjt737A4UxD/4+y7wFaLukn7QeiDVJf+fu/1BiewAAoBzmZgBJbRcA7v6SpPd1MBcAAFACczOAlaANKAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGSm7EjDQFYlW+7p+XckdBH2O64lFAmq1uG4eno0bQB85E/dRBoC15JLqUev44Ckr1edfSvWsjzVU/Hxaq8TP1fVUv/zEMgKNer0wllhCQKnXV62aiFeidQDisc3wlympGic/7MEvfCCxcAM67tjvXw7jWzdvKYwtLs4XxrgCAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZYR2AK9yWRMP8jXGbZL0w27lcVmtL0Ip4fDQxONWKOFXaBselNhwftDHFyU1ULyZ23r11ABK/7qCjdrIlNoCrlEmqBU8Axd3w089mqadiS7WsrxY/q1UsfsZLrSczdWY6jM8vpJ7Li63fvD6+g8enX/FqMqmjHh/UZj2eQEdGio9cNfEb/fAffjiMP/zfHg7j/Wp90Gt/JSbPnulQJm+1GMSiRwJXAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJARCgAAAAAgI6wD0OdSFdrNu8ptf/R0ccwTrYbPx42Kk4L2zhofi8c2gjUEJGlm9em8YS4OW2INArM4uRu2F//g03PxQZ9M5Fafj+P0+gewvOLnpVq0SEDiuTgRVmqWGxkofsJdaMbPaOdPXgjjP/nh/xHGY6mfLJ7E3v7um8L4juuL49s2bgjHJqYgmcXzTHWkePsjicVm3nvH+8J4P68DcPMHPlgYe+d1N4ZjbWQkjP/64YfC+OnjRwpj4xu3hWPbxRUAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZOSqaQP6tu3lxr92sjN5rLVEN6+gsVsrHnQqS3RY0+ZUq7ES5WUl1WEtse2Ls+3vO9ndbWg8DA+k+oiquH/q6Gg8cvJYHE88HABgGSZZcDrgxZNBap6oetwvulaLn8wrNlgYG6wshGPLtflMSTVVng6jLz37VNvxG991Wzh23cZrw/ie63aE8cmJycLYyOb14VjVUqeVqbOS3s1iUavPgfG4zaclThw+9OEPh/Enn3yiMDa2Lj7nuH7XzsLYqy8fLowlT9HM7H4zO2VmBy+5bZOZPWRmL7T+35jaDgAA6AzmZgBlrOQ12u9Iuvuy274g6RfufrOkX7S+BwAAa+M7Ym4G0KZkAeDuv5R07rKb75H0QOvrByR9osN5AQCAAszNAMpo913a2939eOvrE5JKvgMfAACUxNwMYEVKdwFyd1fwaRgz22dmB8zsQNl9AQCAtNXMzXNzZbomALgStVsAnDSzHZLU+v9U0R3dfb+773X3vW3uCwAApLU1Nw8Pxx1OAFx92i0AfibpvtbX90n6aWfSAQAAbWJuBrAiyXUAzOz7ku6StMXMjkr6kqSvSPqBmX1a0iuSPtXNJDthrt7rDLrj2EQc33pNHK8Ut1hWsESAJKmaKB9T7fRVK77H8Mi6cGizeTGM+3DcSzjs1J9IfGhTfIdqI/G224niRScOvRI/UOfjttcAMtHRudmkykAQ9+LnPAvWCJCkSr3cO42jdQaeeuTxUtu+Ur146GAY3/n2ctvfdV3xHLZRiXUA+tjmt90Qxivrh4NgYuOJZSEscbr9/g+8vzDWaMZraahZvLaCBQsyJQsAd7+3IPQnqbEAAKDzmJsBlFH6Q8AAAAAArhwUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZCTZBvRK8Vpxa/UrWqoXf1kjVtw/trJxNBw7PjKe2HrcGHehMV0cnAs79auS2PZQchGCYr4p7uM/MpqomxN9r//5sfhnA4C15C4tBkuQVIunCTUb8ZPtxbnJeN+JxU1mZ4ufL3/96H8Px+bq2EvxOgFajNfJWZwtXmfHG3FP+oOPPxvvW/G+u+kDe4t77UvSQCV4oKfOKRLx1KvtjeC8wRLbDpbpCPPiCgAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZOSqWQfgajWTiK+bT20h6GsrqbqleKWB7Tu2xZuOW/FrMejfLEnVhXXFwUpxH+KlXcfb3jQT9xo+N1ocn1O8qMRs4pfy5K/jvtYA0E/M4l7/kbovhvHXjr0Sj58r83wZ7xvLO/bqoUS8OHb2eDw/njp9up2UOqM2EIbHN2+Jx4f99lOvlydOiBLN/JsWLMSRWHtBHuw7CHEFAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhDegVbuTaOD66YTCMDw1EratSPUYTD59monVVJWiLVVsfDh0Z2xBv+5o49+pkcSuzQ6fjtnQvHIl3DQBXEndpMeqcHLT6nJmcDrc9fTF+Pp29cC6MN5OtrrGWTpx4KXGP3r2uPDiyKXGPuNdtlHkz0eWzYvEdUuNDUZvPpTu0FeMKAAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhHUAViBVJSW63ZcymPgNbRxPjA/7/EuNRnF/57Mnz4Rj14+ti7ddD8NSrXiNgtpIsEaApJFrEwsgJBw5VZzc4uLZeHDq5wKAK4qHvf6bjdnC2JEjL4dbnr1wOoxPTk2E8ZOvFa/Zgn7UzTOiWG1wNIx7fFohqwZne43UzxVv3FPHJVgoILmEQJtrDCSvAJjZ/WZ2yswOXnLbl83smJk91fr38fZ2DwAAVou5GUAZK3kL0Hck3b3M7V9399tb/37e2bQAAEDgO2JuBtCmZAHg7r+UFK/VDQAA1gxzM4AyynwI+LNm9nTrMuTGjmUEAADaxdwMIKndAuCbkm6UdLuk45K+WnRHM9tnZgfM7ECb+wIAAGltzc1zc8Uf8gVwdWqrAHD3k+7ecPempG9JujO473533+vue9tNEgAAxNqdm4eHR9YuSQB9oa0CwMx2XPLtJyUdLLovAADoPuZmACuVXAfAzL4v6S5JW8zsqKQvSbrLzG7XUvfRI5I+08Uc+16qihpKHOWxoHXt9l3x2EGLe8/OzM2H8eZicQPZZiPe9/RUfNl4YHQ4jI9tWV8YqwzEawyMjY6F8ZQL08XN/I+8UmrTANB1nZyb3V3NYE2Y184Ur41y8ujz4bbnpybD+MRU8X6B1XjXu28I41aNG+ZHUU+d6CUXGYjjFuw91ea/Etwh2muyAHD3e5e5+dupcQAAoDuYmwGUUaYLEAAAAIArDAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZSbYBhdRMxONu99JoqsyK2iAnxlYqqb3HpheDXv6JH7yaitcTCwnUgx9uMB46Pxdvu576pQEAJEnNpmv2YvGaMedeK14cZX467vM/NZ3q81+8JguwGtff/I4wbonzKQtOehJLCMiaqT7/cdyD5MxSJzTRtotjXAEAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARmgD2geijpf1s/FY2z0Uxkdr68J4pTZVGJucmIi3PTYQx9etD+MjteLxm7duC8cOWdyT6+jp4p9Lkn77woUwDgC5aDQWNTlxsjB+/tiZwli9Xtw+VJIafgX3ZE7Mn6F6PAdh9d7xgT8O43GjTWkocY+oc3mqhagSbULViDdQUbV438lm9MXx6CfmCgAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIR1ADpgKHEUh4rbuy4Zbn/fqT7/m7dfE8YHa8X99ivD0QoF0vT5c2G8MTcTxquK1xGIzHs9jD/z1MG2tw0AOZmfndXzTxc/Z752/Hgw+srt879pxw1h/F9+7O62tz03Fc9/vzn0XBh/6eCTQTSe/0ob3VAcm4nXB+qmd9+6J4yn+vynDlv4iniwRoAkNevxQgAVj+NRtJJ8rT61CEHRdgEAAABkgwIAAAAAyAgFAAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGWEdgA6YSfSWHYtb9WtdsA7Atq2bw7HNZP/XuEfzQn2uMDaseB0AVeOHT6o7dLQ8Qv1iouluwvnzXe6TDABXicXFxauy13+qz/+73/mONcrkrW69ZXcY37ptfWFsdHQoHFtJvbSbiA8Es/M//Tpan0C6eOJoYuftG0id7zQS5w2WWJQp2HzT4zUGPLEOQLJXf3OxOJbcdnuSVwDMbLeZPWxmz5nZs2b2udbtm8zsITN7ofX/xq5kCAAA3oS5GUAZK3kLUF3SX7n7rZL+QNJfmtmtkr4g6RfufrOkX7S+BwAA3cfcDKBtyQLA3Y+7+xOtr6ckHZK0U9I9kh5o3e0BSZ/oVpIAAOANzM0AyljVZwDMbI+kOyQ9Imm7u7/+psETkrYXjNknaV/7KQIAgCLMzQBWa8VdgMxsXNKPJH3e3Scvjbm7q+ATDu6+3933uvveUpkCAIA3YW4G0I4VFQBmNqClJ5jvufuPWzefNLMdrfgOSae6kyIAALgcczOAdq2kC5BJ+rakQ+7+tUtCP5N0X+vr+yT9tPPpAQCAyzE3AyhjJZ8B+ENJfyHpGTN7qnXbFyV9RdIPzOzTkl6R9KnupNj/gu6tkqSLU3F8JNFuP+KJvrenL1wM48WrAEjNC9Ph2LMXEj95onX01uAOW5pn4sGK+yADwFWuw3Nzd3r9x93T08p0QL9w/OUwfnZD3Be+GqzRMziU6CmfcGFqvtT4Xrl+dCGMP9fFfZ87/GgY3/T2O8P4YnJ5ofbXxk09GhqJR/LFucSJYsCCvBte/HedLADc/Vcq/hv+k9R4AADQWczNAMpov9wBAAAAcMWhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEVtaKHCNdma2djvrIwOJ+IZ1xbE9120Ix14zPhLGZxOtyqZePV4Ye/rlZM+sUkaD2Afu2BmO3bJ5fRifPD8Zxn/x+LEwjtVz97Id/wD0QDfn5ms3lRt/4lxn8lhO8gkr6DZdTQweT3SqttSJQSDVenw60UX7msT4a6/fWBg78er5cOyF7nSTlSTt3BzHBwbjgz6fepQHv9Oaxa+X22D8gEjtenEuasoeGxzbUhg7cfSc5ucWl02OKwAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBHWAVgDg4ky65qx4tie3fE6AAO1uM//9NREGO92r/92RWsESNIf3RmvEyALFleQdPDwK4Wxo2dnE3vHclgHALgydXNu3vuut5Uaf+Dwa8XBVEP8LrJaHK/GU7OGh+N4M+inP7MQj9V8In6FSr1iPZaagRJrMwwFpw2pfaceiouJtRmqwQ4qibwjk3NSvbH83MwVAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSEAgAAAADISKKTbT4Go2CiTFoI+vWuRGO4uGHw3GLcp7+WWAfg1fP92ec/ZSYRX2wk2lZX417+N+zaVhg7erZ4jQAAwCqUfZnxmiB2oeS2U4Lp1evx0HqiJ/1MYmpuRuMTawwkj3nJc5ZeSaU9lTgtsMT6CdNTq0rnTVKPB6VW2gh+uPHE0kQDbe6XKwAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBFzj5uTmtluSd+VtF1LHUX3u/s3zOzLkv6tpNOtu37R3X+e2FaqE2rPhOsAJFZLWEj1fy1h09a44a8n1gk4P9HJbPrHdcVt/CVJN+zcGcabzeJf6nwzbhb86DPH451nyt0Tna8BdMqVMjff8e6wS3nSk8cWi4MXS206/RJo9IxWoq/7inTz5dngkEoql/toHB7bEMcvvlZi3ym9XPkqtSRTmcdaIl40N6/kcNQl/ZW7P2Fm6yQ9bmYPtWJfd/e/XcE2AABA5zA3A2hbsgBw9+OSjre+njKzQ5Lil1cBAEDXMDcDKGNVF5nMbI+kOyQ90rrps2b2tJndb2YbO5wbAABIYG4GsForLgDMbFzSjyR93t0nJX1T0o2SbtfSqxBfLRi3z8wOmNmBDuQLAABamJsBtGNFBYCZDWjpCeZ77v5jSXL3k+7ecPempG9JunO5se6+3933uvveTiUNAEDumJsBtCtZAJiZSfq2pEPu/rVLbt9xyd0+Kelg59MDAACXY24GUMZK2oB+SNI/SXpGbzSH+qKke7V0idElHZH0mdaHkqJt9W0b0FyNDBXHZufXLo9O+9B7dsR3qESNX+O2da9MToXxV18+Ge/7KkUbUGDt9MvcvDHuVK09t8TPpy+djHtSXjgXBMu22izjSl5FqcxxG4nDY4lPnGy/ZrjtXb90eC6+Q6ole2qG6uXjqYvabgPq7r/S8oct7CsMAAC6g7kZQBlXcg0LAAAAYJUoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyEiyDSjKq5Q4ys1UX9uE97/nxjBeHSxu4txIbPvQwefDeC/XEfjVM2Hba90ZrBMwaHHf6vF168L4hs3TYXzi7MUwDgBXinfesrPU+AvnjsV36Nfe7P2aV7fNxuH50Tg+XttUYuevxeHUahasRPUmXAEAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIywDkAfGBiL+85HNo9tDOMXZuKmvZs0Uhg79PyL4dhe9vkv69HEOgEAgDWQaz/9q1T9bBw/nOjlPxwsszNafLoiSZpZjON4M64AAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGWAdgDTTrcXz+QnHz2re9bVupfc/Oxjt/7OWg1z/9mQEAgYnmhV6ngCvITGKdAAW9/GcmO5pK9rgCAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIyYu6/dzszWbme52LI+jp9L9M2i1Sc6xN2t1zkAWL0yc/O2XeX2fepoufEAYkVzc/IKgJkNm9mjZvYbM3vWzP66dfsNZvaImR02s783s8FOJw0AAN6KuRlAGSt5C9C8pI+4+/sk3S7pbjP7A0n/QdLX3f0mSeclfbp7aQIAgEswNwNoW7IA8CXTrW8HWv9c0kck/bB1+wOSPtGVDAEAwJswNwMoY0UfAjazqpk9JemUpIckvShpwt3rrbsclbSzOykCAIDLMTcDaNeKCgB3b7j77ZJ2SbpT0i0r3YGZ7TOzA2Z2oM0cAQDAZZibAbRrVW1A3X1C0sOSPihpg5nVWqFdko4VjNnv7nvdfW+pTAEAwFswNwNYrZV0AdpqZhtaX49I+qikQ1p6svmz1t3uk/TTbiUJAADewNwMoIzkOgBm9l4tfZCoqqWC4Qfu/jdm9nZJD0raJOlJSf/a3ecT22IdAOAqxToAwNrpm7m52vbIJY2S4wGEiuZmFgID0BEUAMCViQIAuHq1vRAYAAAAgKsHBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmppe/SUWckvXLJ91tat/Wbfs1LIrd29GteUv/mttq8ru9WIgC6rv25eW3bePbr86VEbu3o17yk/s2tY3Pzmq4D8Jadmx3ox2XI+zUvidza0a95Sf2bW7/mBaD7+vXvv1/zksitHf2al9S/uXUyL94CBAAAAGSEAgAAAADISK8LgP093n+Rfs1LIrd29GteUv/m1q95Aei+fv3779e8JHJrR7/mJfVvbh3Lq6efAQAAAACwtnp9BQAAAADAGupJAWBmd5vZ78zssJl9oRc5FDGzI2b2jJk9ZWYHepzL/WZ2yswOXnLbJjN7yMxeaP2/sU/y+rKZHWsdt6fM7ONrnVcrj91m9rCZPWdmz5rZ51q39/S4BXn1/LiZ2bCZPWpmv2nl9tet228ws0daf6d/b2aDa50bgLXD3LziXJibV58bc/Pqc+vq3LzmbwEys6qk5yV9VNJRSY9Jutfdn1vTRAqY2RFJe9295/1fzeyPJE1L+q6739a67T9KOufuX2k9QW9093/fB3l9WdK0u//tWuayTG47JO1w9yfMbJ2kxyV9QtK/UQ+PW5DXp9Tj42ZmJmnM3afNbEDSryR9TtK/k/Rjd3/QzP6TpN+4+zd7lSeA7mFuXlUuzM2rz425efW5dXVu7sUVgDslHXb3l9x9QdKDku7pQR59z91/KencZTffI+mB1tcPaOmBuqYK8uoL7n7c3Z9ofT0l6ZCknerxcQvy6jlfMt36dqD1zyV9RNIPW7f35LEGYM0wN68Qc/PqMTevXrfn5l4UADslvXrJ90fVJwe7xSX9o5k9bmb7ep3MMra7+/HW1yckbe9lMpf5rJk93boMueaXPy9nZnsk3SHpEfXRcbssL6kPjpuZVc3sKUmnJD0k6UVJE+5eb92l3/5OAXQWc3M5fTPHLKPnc8ylmJtXlVPX5mY+BPxWH3L390v6mKS/bF1S60u+9P6tfmnj9E1JN0q6XdJxSV/tZTJmNi7pR5I+7+6Tl8Z6edyWyasvjpu7N9z9dkm7tPRK4C29yAMACjA3t6cv5pjXMTevTjfn5l4UAMck7b7k+12t2/qCux9r/X9K0k+0dMD7ycnWe9Zef+/aqR7nI0ly95OtB8SYZBcAACAASURBVGpT0rfUw+PWeq/cjyR9z91/3Lq558dtubz66bi18pmQ9LCkD0raYGa1Vqiv/k4BdBxzczk9n2OW009zDHNz+7oxN/eiAHhM0s2tTzEPSvpzST/rQR5vYWZjrQ+ByMzGJP2ppIPxqDX3M0n3tb6+T9JPe5jL/+/1P+CWT6pHx631oZlvSzrk7l+7JNTT41aUVz8cNzPbamYbWl+PaOlDgIe09GTzZ6279c1jDUBXMDeXw9wc58HcvPrcujo392QhsFY7pf9dUlXS/e7+v615Essws7dr6ZUFSapJ+rte5mZm35d0l6Qtkk5K+pKk/1PSDyRdJ+kVSZ9y9zX90E9BXndp6VKZSzoi6TOXvK9vLXP7kKR/kvSMpGbr5i9q6T19PTtuQV73qsfHzczeq6UPElW19KLAD9z9b1p/Dw9K2iTpSUn/2t3n1zI3AGuHuXnF+TA3rz435ubV59bVuZmVgAEAAICM8CFgAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGamUGm9ndkr4hqSrpP7v7VxL39zL7A9C/3N16nQMA5ubXVavVMD4yMtb2tq3ky6dzMxfD+GK9UW4HV6jBYBYZGYlPWc3iKcgVxy34pSbHKv4TSv2BRVtP/VwK4hdn5zW/UF/2Dube3t+9mVUlPS/po5KOSnpM0r3u/lww5qp8kgFAAQD0A+bmN2zYuDmM33bb3jAenXjVhlNPd80w+sITj4Txo2cmEtu/Ol03VBy7/b3bw7GV2kAYb1biqq0yMFoYqyve9mDiFL8eRuNX42u1xGv1wc/10K9/q3MXLi77YC1Tw94p6bC7v+TuC5IelHRPie0BAIBymJsBJJUpAHZKevWS74+2bgMAAL3B3AwgqdRnAFbCzPZJ2tft/QAAgJVhbgbyVqYAOCZp9yXf72rd9ibuvl/SfunqfZ8hAAB9grkZQFKZtwA9JulmM7vBzAYl/bmkn3UmLQAA0AbmZgBJbV8BcPe6mX1W0n/RUqux+9392Y5ldplrduwJ4xeOH+nWrlGgNjIexre+7V1h3INP1bvmw7Hzs3GHhcbka2F8avpkGAeAK9Faz839bOL82TBery+GcasEnV/m4osm5yYnw/hrmXb5GU7Et2wvvoeluuFUE11+qkGLIUm1oeLfdy3xenkz0bXV6vE5S22weN+VRM/ZWhCOelWV+gyAu/9c0s/LbAMAAHQOczOAFFYCBgAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJARCgAAAAAgI11fCXg1Rtdvb3vs2NjmMH7xYtwODG+VavO5afv18figrZUk1evFMRuM9z1s02F8Tm8L45o+EwQT/byuYANWLYwt+tX7cwPA5c5Ox604R4aK20Y2gvlLkiYvxG1A46aQ5WwYjOPNhTge/Whl8771urgR6Lqtu4qDg1FTS0mV+DXtai0+J4malNZTS+UlXk4fqsUtZ2u14l9aqg3ogBU/Ti2Y87kCAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZ6at1AMIOszOz4chGc6bDuSClvjgXxgdqo2G8YsUPv4FES3qvxP18FzUfb+Aq7fWf6JKsq/XnBoDVmp+P54mFIOzNuDH8qy8faielFdk4FD/Tb9jznjC+Z9tYGF+sFy8U4M3EAgjRAj+SKoqP23CwBJBV4p/bEscltYaBN4vnR/d4tCXOSapD8foHpuJ+/QOJseHMH4S4AgAAAABkhAIAAAAAyAgFAAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGemzdQCKLSZ6sM7NxusEoA0W97X1RD/fhXq8NoNZ0KDWg2bAkuqJPv9nzx8J41er+DciJVpXA0A2fv/K4TC+a9dNhbFTr5zsdDorluzzHzXTV3q9mAENFsbqqW76wfo+klStxXsv7oYvaSCewAaTr2kn1gnw4r0vxJlpJLHtylBinYDgmHvimFqUWnCexRUAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMhIqXUAzOyIpClJDUl1d99bZntD69e1PXZm8nSZXWM5g+Xqw7rPxXcIWvp6ol/96ZPH4jvMno/jmWpG/YIba5YGgC7q9Nx81ZqP15M5+mK8TkAs0S+/hJd/93QYH3z3HWF8x0R86tdQvTDmzeKYpEQjf6mZuEOjuB2+tBifkywORIOl4MeSJA0EJx7VxOIJzcpofIeFOPfKYPE6AZ5YY8CSKzssrxMLgX3Y3c90YDsAAKAzmJsBFOItQAAAAEBGyhYALukfzexxM9vXiYQAAEApzM0AQmXfAvQhdz9mZtskPWRmv3X3X156h9aTD09AAACsDeZmAKFSVwDc/Vjr/1OSfiLpzmXus9/d9/IhJAAAuo+5GUBK2wWAmY2Z2brXv5b0p5IOdioxAACwOszNAFaizFuAtkv6iZm9vp2/c/d/SA8rbgFVqxbXI1aNUx3bsjWMz0zEbSG9nugPhbdo1uNenReO/T6xBY55xw3G7cCKG41JC41E71UAV4I252a8VZnnxERLSi20veWb3hW3+Ux59cyFMD5zvrh5VMMT8/b8xTC8eyQ+LsM7i8/lGhu3hWOrG6IZThoNzjElqRG102ym+oDG8UZifm0GD4eBVGvVoONs1FK97QLA3V+S9L52xwMAgM5ibgawErQBBQAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJARCgAAAAAgIxQAAAAAQEbKrAPQcRaks354fTy4UW7fi7PzbY9dmE3svBn3xe1X9YmzYfyC4ji6YGhLGB5ObmAiiLEuAwC8of1e/al1ADZufUcY3xysbTSafOl2MYz28pn+hMfHdOHY6cLY5sn45xpr7gjjja3x76RW3RDGI9VoDQFJ9dRpYrCu0rEXj8aDp4uPy8LF4uPNFQAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyEhfrQOwbqy41//AyLpw7PrqQBzfVNxTV5JUKa6FKon+rqdOnYy3PRf3y584fSoeD7Ss3xj/HaQq+uZk8ToAc23kAwB5SvX53xPGd9x0UxhfNxw9myee6SvNMDx47TXx+OeKQ1Pnz4RDFzQWx+fidZFOarYwVvfimCTpxeNh+INb3x/Gtwex2lhilZ3B+HQ6PkOV5s8ULxTQCPr8S5KreB0rV/FjgSsAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJCRHqwD4IWRwY3F6wDEHXel5I9Si+MerSPQLO7PKknrN20M41PnwrBUCfriNuOeucjLzNRUGB9PjL+4WO9cMgCQmpxTTznFpwQri/fIxuuuC+M7dsZ9/gcT5yTh8kOJY1L2kE3Xi887FubiNQZSv/AJxedTodRiNYn4f/+HJ9re9TW7R8L43X98TxjfORqtMiApfjiFTj57KIhGa1wBAAAAyAYFAAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGelBG9DiFlL1qeL2UZWRarxZi1tTpSqdZnCPZqVcnTQ4HDdn3LizuP/T+Vej9k7A6tQ0WhhraGYNMwGAPjde3LpxvBbP6zXNhnFvxG0l54NumfPzE+HYQ08nzhsWz8dxvMWFV+Pf53/951+H8Y/9y7vC+J7RXYWx295xazh29tXXCmPV08XnzskzWzO738xOmdnBS27bZGYPmdkLrf/jRvgAAKBjmJsBlLGSl7a/I+nuy277gqRfuPvNkn7R+h4AAKyN74i5GUCbkgWAu/9S0uVr2d4j6YHW1w9I+kSH8wIAAAWYmwGU0e6b27e7+/HW1yckJdY4BgAAXcbcDGBFSn8I2N3dzLwobmb7JO0rux8AALAyzM0AIu1eAThpZjskqfX/qaI7uvt+d9/r7nvb3BcAAEhjbgawIu0WAD+TdF/r6/sk/bQz6QAAgDYxNwNYkeRbgMzs+5LukrTFzI5K+pKkr0j6gZl9WtIrkj7ViWSacxcKYwsaCsf63HwYr66Px6ta3Ct17mLc/3WhEffUtcQSBudfeyG+A9AyNjIWxguv979uIX4sx6IHctC0GkDHreXcHFro+h56Zt30meLgps3h2JnZ+DlxdGA6jJ9ZKF6X5dgzz4Rj1YzPh9B5/+K2O8L45sQyO43KZGFsdqL43FiSto0Xr+9Tqxa/zp8sANz93oLQn6TGAgCAzmNuBlBGuSVuAQAAAFxRKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMhIsg3oWpo4+0phbMOmdfHYc1NhfGQ+0Yxfw4WR+kBcJwXteiVJ586cjO/QqMdxoKUyUu5PdtPotYWxU8dPhGNrA8V/gwtzQb9sAOgBS8THw7VNpA0b9gTRwXDs0GTc5/9sYto/dvRwcfAq7vMfrdgUr7yQtjMRT50lRl47+GQYH//gH4fxiaDXf/3w78Oxp6aLT0LrjWZhjCsAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJCRvloH4PyFc4WxizPFMUmaS7XFPZ+IRw2DU81hGyXjwAo1Z8uNn695cTDo8y9J1cHiDs02z2sJAFZvT6Kf/jaNF8ZSff5ThodGwvhAGIuf847X4m1vqMdP5idHxwpj9enJcGzxqkYrsxDEirvKL0nNBG9PxN8WxMr06ZfSp2LB7Kjkak0vHQnDLybikeHEb3QgPDLFjzNmbQAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSkr9qANoP+UguLZTdeYixtPNEnmvXXSo2fOlP8YE79iczNFLela/JHAly1onabcbNL6cZEm8/1Axva3rcS2x4OGztKtS6+Bro+eY/4yH3YdhfGLPFs7WEjT8kSfdHj0bHUPHKhxLbLih8NUpnTzLItaefCWBSNRe1LuQIAAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABnpq3UAIgPr436/6wbjrrtnTpzpZDpAT0xNzPZs33F/5zILbQDoZ1HH+lS/+1rquWEx1Z09MJCIJxq71xO5zc8H/dcn4rVPdlc2hfHaWOrIFSd/IjEy1ec/1bN+KIil1ghIbTte9SEeX7aPf9QTX5LKzK7Fq+SU99tE/GKb201eATCz+83slJkdvOS2L5vZMTN7qvXv423uHwAArBJzM4AyVvIWoO9IunuZ27/u7re3/v28s2kBAIDAd8TcDKBNyQLA3X8p6dwa5AIAAFaAuRlAGWU+BPxZM3u6dRlyY9GdzGyfmR0wswMl9gUAANKYmwEktVsAfFPSjZJul3Rc0leL7uju+919r7vvbXNfAAAgjbkZwIq0VQC4+0l3b7h7U9K3JN3Z2bQAAMBqMDcDWKm2CgAz23HJt5+UdLDovgAAoPuYmwGsVHIdADP7vqS7JG0xs6OSviTpLjO7XUttWY9I+kwnkhnaUNzrf/f2m+PBcUveJNYJQKekquort2N+qgM0gLWylnNzZDIR35Lovn5Bp8P4dBRMNH5P9aS/JhFfp51t7/xoM45vmDqb2Hvxka3rZGJsLHXiFx231NIL86vM5XLRLBOsyrAiZfr8p04xLyTiqdyjR0u7ff5TkgWAu9+7zM3f7kIuAABgBZibAZRRpgsQAAAAgCsMBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABlJtgFdS9dtuaUwVqvGtcpiI+41TJ9/QBpbN1oYa8rDsbNTZbooA7hSzQSx4meUJamO9WOrzKWT1m17R9tjR86Mh/HNm9aFcUs8n07PF68DUPaZONXLP1oHILXuQ0qZdXAmEvGSy0GFyq7vE5+hKjH7dgdXAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJCRvmoDutiYK4z5Qty46vDzz3Y6HaAtZdqclXXNurj1XG24uMHbXKpR2VQ7GQG4mkUtQlcSf9d79obxwcHgOW1gYzh203Bi54nnvAvHzhYH5+Kf7OJC3DAz1f60HjS1PJ0YG2S9IrtKjE21CV1IxKtBrGyrzDJzcz+/Wl4NmvE2VHxe3c8/EwAAAIAOowAAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARvpqHYAjLz7f6xSArhsMYsWdn5eMJ+K1wWTj60ILU6kOzQCwOu/StjA+nnxWK9b+s92SiRNxx/zmCwcKY/PJrvTx6VVdm8L4z3U0sf3uiVY4SK1fEK/YlBYd1WiNAEkqXuVmSZncUttOuZCID4XR4j7/S6LfSvG8zhUAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMhIch0AM9st6buStmupRet+d/+GmW2S9PeS9kg6IulT7n6+e6kCV4ZUv+Coo29qHYBmIj45ezFxj2Dfi3NtjwWwtq6cuTl+XlmYSaw/EjSln5t+Oh7bjHv1L55+NYyfCnqop56LPRgrSYfDbvu9NV9ibGqdgNRJZ9TrP7XtsmaDWGqVnDOJeL1UPD4zaGi6rbEruQJQl/RX7n6rpD+Q9JdmdqukL0j6hbvfLOkXre8BAED3MTcDaFuyAHD34+7+ROvrKUmHJO2UdI+kB1p3e0DSJ7qVJAAAeANzM4AyVvUZADPbI+kOSY9I2u7ux1uhE1q6DAkAANYQczOA1Up+BuB1ZjYu6UeSPu/uk2ZvvNPZ3d3Mln2znZntk7SvbKIAAODNmJsBtGNFVwDMbEBLTzDfc/cft24+aWY7WvEdkk4tN9bd97v7Xnff24mEAQAAczOA9iULAFt6OeHbkg65+9cuCf1M0n2tr++T9NPOpwcAAC7H3AygjJW8BegPJf2FpGfM7KnWbV+U9BVJPzCzT0t6RdKnupMiAAC4DHMzgLaZe9wnt6M7K3gvInA12bHiT9a81XSiWfBU+5vuOndPLYEAoA/1cm6+scTYVG/21Loq3XzCOpuIX62rrmxKxAcS8WgdgPFV5nK5qM+/JMWrQvRS6s06Q0FsTu7NZR/qrAQMAAAAZIQCAAAAAMgIBQAAAACQEQoAAAAAICMUAAAAAEBGKAAAAACAjJRoWAj0zvrtW8L4us0bwvjJF4qbtNUXz4djU63jRtYl7hA4Hu8aAK4qLybi20psO9FVWYOJeLPEtlMtSMsZTsR712T0XCKe+n1Gx/X5VeZy9Yibp27dfl1h7PzZI4UxrgAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyAgFAAAAAJARCgAAAAAgIxQAAAAAQEbM3dduZ2ZrtzNc2RKl6c5bbgrjw2Mbw/hAo1oYO/zM78KxlXrcrL9Soqye627z6K5y99QSCQD60JU6N6eecOLu6eUWQkqtA5A6oIvJPQS9/mtD8dD6fGLbvVsnIF/xOUm1Wvxo3fy2TeHYj/+rjxfG/q8ffU9nTp1Y9k+FKwAAAABARigAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkJEybXCBnjl3+vdh/L3Xva/tbd/0nriD80tPPhLGr+Re/rHo6SLVFRsAOivVa38hEe/ms1bphRVSvf5DPB933u4wOjwSrzoxVItPt4c2FMc//q/uDsduGyleM6JmxatlcAUAAAAAyAgFAAAAAJARCgAAAAAgIxQAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADKSXAfAzHZL+q6k7Vpqbbvf3b9hZl+W9G8lnW7d9Yvu/vNuJYq8DJQsTW2h/S7MQyNxvMSmAaAjmJvLa/Zw3yOJ11+HBq4pjDU2xpPU1GuvJvae6zoBG8Po2NiGwljV4j7/VYt/n1Yr7scvSXvf+c7C2KahwXjf0X6D2EoWAqtL+it3f8LM1kl63MweasW+7u5/u4JtAACAzmFuBtC2ZAHg7sclHW99PWVmhyTt7HZiAABgeczNAMpY1RstzGyPpDskPdK66bNm9rSZ3W9m8bUVAADQcczNAFZrxQWAmY1L+pGkz7v7pKRvSrpR0u1aehXiqwXj9pnZATM70IF8AQBAC3MzgHasqAAwswEtPcF8z91/LEnuftLdG+7elPQtSXcuN9bd97v7Xnff26mkAQDIHXMzgHYlCwAzM0nflnTI3b92ye07LrnbJyUd7Hx6AADgcszNAMpYSRegP5T0F5KeMbOnWrd9UdK9Zna7ltqPHZH0ma5kiKvWjut3FMYuzJwLx24aG4837icTe4/bal2tRos7y2lg9O3h2MrgaGFs8vjhdlMC0B7m5ivYbGoOWpwrjs0kelXrSu5VHTSutK3hyNrwpjA+PBy34qwEv5NqomdsvGXpHe/YHsav3Vb8s83MzoRjo2ijWZz4SroA/UrL/2z0FQYAoAeYmwGUwUrAAAAAQEYoAAAAAICMUAAAAAAAGaEAAAAAADJCAQAAAABkhAIAAAAAyMhK1gEA2hL1+ZekG259T2Hs6Iu/ize+OB2GTx75bRjfvue2wthAZTjedxdt3Bkfs/PHjscbSPxFV0fWFcZGgz7/S4o7HU8nuyADAN4wH0Zn6xeLgxNTiW3Prj6djlkfh2vBYjSShgbGioMDidesLV7/wMuc8qZ2nYrXhsL48TOThbH4rEB66fDLhbHZmeLHAlcAAAAAgIxQAAAAAAAZoQAAAAAAMkIBAAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjrAOArnn///jhMF7RQGHsmttGwrG/ffJXbeX0urn5uVLjy7jjX8THJTI/s5i6RxjdveO6IBr38j9/LrFrAMAKxT3rpWAdgK4bLw5VE33+B+O5e3gg7ocfHpXkcjPxHer1+Jhbtf19VwbiO0yeb39thqNHT4bx2YmZwtjiYrMwxhUAAAAAICMUAAAAAEBGKAAAAACAjFAAAAAAABmhAAAAAAAyQgEAAAAAZIQCAAAAAMgI6wCgbeNbdyTuMRhGLXj4VQY2hmNrNhzGF5To8z9zIY530eHDz7Q9dmbqTBgfXbctjG/YcmPb+z5/+uXiYKqlNQCgT2yNwwPFc/fgQDz3mqWa9dcT0eLzgtQJa6MZT0RVb4TxxUbx+KFq/Hr5YiP+uU+cPx/G52cmi4MLqQm2uNd/NDlzBQAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZSbYBNbNhSb+UNNS6/w/d/UtmdoOkByVtlvS4pL9w94VuJov+sn3XtWH8xNHfh/Frd11XGHvlxaNt5fS6M6fjtlmvvPTbUtsvY+p43MqzjC3bd8d3aEwVhs7FXco0c3G6MNZsxu3VAHQWc/PVbqjNmCQbD8OVWvzar1WKTw0bjajlpKREu0ylpopKcZvQ+lzcQjTVgNSr8Slv8GOnf65qfM7R8Pi4XZwrHj9YiX+ygVq1OBgMXckVgHlJH3H390m6XdLdZvYHkv6DpK+7+02Szkv69Aq2BQAAymNuBtC2ZAHgS15/6W+g9c8lfUTSD1u3PyDpE13JEAAAvAlzM4AyVvQZADOrmtlTkk5JekjSi5Im3P316zFHJe3sTooAAOByzM0A2rWiAsDdG+5+u6Rdku6UdMtKd2Bm+8zsgJkdaDNHAABwGeZmAO1aVRcgd5+Q9LCkD0raYGavf2Ril6RjBWP2u/ted99bKlMAAPAWzM0AVitZAJjZVjPb0Pp6RNJHJR3S0pPNn7Xudp+kn3YrSQAA8AbmZgBlJNuAStoh6QEzq2qpYPiBu//fZvacpAfN7H+V9KSkb3cxTwAA8AbmZgBtSxYA7v60pDuWuf0lLb3nEFjWxdm4Z+8rL0S9/uO21a+dno33PXUujOcq6vU/MzsZjnUvPuauRG9oAB3F3NzvUm+wGEnEg17/NhyOTPXDT7/3I5i7PXHamFgnoJkYXpkv7odf8bjXfpIvxnEr7qdfqcWJN5TYdnRMFZ/xDNYGwrGWXFxheawEDAAAAGSEAgAAAADICAUAAAAAkBEKAAAAACAjFAAAAABARigAAAAAgIxQAAAAAAAZMS/bV3U1OzM7LemVS27aIunMmiWwcv2al0Ru7ejXvKT+zW21eV3v7lu7lQyA7mFu7ghyW71+zUvq39w6NjevaQHwlp2bHXD3vT1LoEC/5iWRWzv6NS+pf3Pr17wAdF+//v33a14SubWjX/OS+je3TubFW4AAAACAjFAAAAAAABnpdQGwv8f7L9KveUnk1o5+zUvq39z6NS8A3devf//9mpdEbu3o17yk/s2tY3n19DMAAAAAANZWr68AAAAAAFhDPSkAzOxuM/udmR02sy/0IociZnbEzJ4xs6fM7ECPc7nfzE6Z2cFLbttkZg+Z2Qut/zf2SV5fNrNjreP2lJl9fK3zauWx28weNrPnzOxZM/tc6/aeHrcgr54fNzMbNrNHzew3rdz+unX7DWb2SOvv9O/NbHCtcwOwdpibV5wLc/Pqc2NuXn1uXZ2b1/wtQGZWlfS8pI9KOirpMUn3uvtza5pIATM7Immvu/e8/6uZ/ZGkaUnfdffbWrf9R0nnipU0xQAAA4tJREFU3P0rrSfoje7+7/sgry9Lmnb3v13LXJbJbYekHe7+hJmtk/S4pE9I+jfq4XEL8vqUenzczMwkjbn7tJkNSPqVpM9J+neSfuzuD5rZf5L0G3f/Zq/yBNA9zM2ryoW5efW5MTevPreuzs29uAJwp6TD7v6Suy9IelDSPT3Io++5+y8lnbvs5nskPdD6+gEtPVDXVEFefcHdj7v7E62vpyQdkrRTPT5uQV4950umW98OtP65pI9I+mHr9p481gCsGebmFWJuXj3m5tXr9tzciwJgp6RXL/n+qPrkYLe4pH80s8fNbF+vk1nGdnc/3vr6hKTtvUzmMp81s6dblyHX/PLn5cxsj6Q7JD2iPjpul+Ul9cFxM7OqmT0l6ZSkhyS9KGnC3eutu/Tb3ymAzmJuLqdv5phl9HyOuRRz86py6trczIeA3+pD7v5+SR+T9JetS2p9yZfev9UvbZy+KelGSbdLOi7pq71MxszGJf1I0ufdffLSWC+P2zJ59cVxc/eGu98uaZeWXgm8pRd5AEAB5ub29MUc8zrm5tXp5tzciwLgmKTdl3y/q3VbX3D3Y63/T0n6iZYOeD852XrP2uvvXTvV43wkSe5+svVAbUr6lnp43FrvlfuRpO+5+49bN/f8uC2XVz8dt1Y+E5IelvRBSRvMrNYK9dXfKYCOY24up+dzzHL6aY75/9q7Y5QGgigO498jIoiNiLYWgq0nsEhlL4goCJbewUYQbMULaKlipV7B0sJCwVZLj2DjWOwoAU0wSnYC8/2qJdnAnyG7j7fZl7U2/90oanOJBuAOWMpTzJPAJnBTIMc3ETGdh0CIiGlgFXgc/KnW3QA7eXsHuC6Y5cvnAZytUWjd8tDMCfCUUjrqeavouvXLNQ7rFhHzETGTt6dohgCfaE4263m3sfmuSRoJa/P/WJsH57A2D59tpLW5yIPA8t8pHQMd4DSldNh6iB9ExCLNlQWACeCsZLaIOAe6wBzwCuwDV8AlsAC8ABsppVaHfvrk6tL8VJaAZ2C3576+NrOtALfAA/CeX96juaev2LoNyLVF4XWLiGWaQaIOzUWBy5TSQT4eLoBZ4B7YTim9tZlNUnuszb/OY20ePpu1efhsI63NPglYkiRJqohDwJIkSVJFbAAkSZKkitgASJIkSRWxAZAkSZIqYgMgSZIkVcQGQJIkSaqIDYAkSZJUERsASZIkqSIfU8WcEj6WE4wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import datetime as dt\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, axes = plt.subplots(2, 2, figsize=(16,10))\n",
    "\n",
    "for i in range(2):\n",
    "    for j in range(2):\n",
    "        axes[i, j].imshow(x_art_adv_def[np.random.randint(10000, size=1)[0]][:,:,:])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow1.1] *",
   "language": "python",
   "name": "conda-env-tensorflow1.1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}