cosine-annealing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, Y_train), (X_test, Y_test) = keras.datasets.cifar10.load_data()\n",
    "X_train, X_test = X_train/255, X_test/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 3)    12          input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   448         batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   2320        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 16)   0           conv2d_2[0][0]                   \n",
      "                                                                 conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 16, 16, 16)   0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 16, 16, 32)   4640        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 16, 16, 32)   128         conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 16, 16, 32)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 16, 16, 32)   9248        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 16, 16, 32)   0           conv2d_4[0][0]                   \n",
      "                                                                 conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 16, 16, 32)   128         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 16, 16, 32)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 8, 8, 32)     0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 8, 8, 64)     18496       max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 8, 8, 64)     256         conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 8, 8, 64)     0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 8, 8, 64)     36928       activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 8, 8, 64)     0           conv2d_6[0][0]                   \n",
      "                                                                 conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 8, 8, 64)     256         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 8, 8, 64)     0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 4, 4, 64)     0           activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 4, 4, 128)    73856       max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 4, 4, 128)    512         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 4, 4, 128)    0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 2, 2, 128)    147584      activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 2, 2, 128)    512         conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 2, 2, 128)    0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 1, 1, 128)    0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 128)          0           max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           1290        flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 296,742\n",
      "Trainable params: 295,776\n",
      "Non-trainable params: 966\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "X = X_input = keras.layers.Input(X_train.shape[1:])\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = X_skip = keras.layers.Conv2D(16, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.Conv2D(16, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.Add()([X, X_skip])\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.MaxPooling2D()(X)\n",
    "X = X_skip = keras.layers.Conv2D(32, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.Conv2D(32, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.Add()([X, X_skip])\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.MaxPooling2D()(X)\n",
    "X = X_skip = keras.layers.Conv2D(64, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.Conv2D(64, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.Add()([X, X_skip])\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.MaxPooling2D()(X)\n",
    "X = keras.layers.Conv2D(128, (3,3), padding='same', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.Conv2D(128, (3,3), padding='valid', kernel_initializer='he_normal')(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('relu')(X)\n",
    "X = keras.layers.MaxPooling2D()(X)\n",
    "X = keras.layers.Flatten()(X)\n",
    "X = keras.layers.Dense(np.max(Y_train)+1, activation='softmax')(X)\n",
    "M = keras.Model(X_input, X)\n",
    "M.compile('nadam', 'sparse_categorical_crossentropy', ['acc'])\n",
    "M.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### search for best cosine annealing max learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "\n",
      "Epoch 00001: LearningRateScheduler setting learning rate to 0.05.\n",
      "50000/50000 [==============================] - 57s 1ms/step - loss: 1.5459 - acc: 0.4381 - val_loss: 1.9256 - val_acc: 0.4286\n",
      "Epoch 2/10\n",
      "\n",
      "Epoch 00002: LearningRateScheduler setting learning rate to 0.1.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 1.0897 - acc: 0.6186 - val_loss: 1.8453 - val_acc: 0.5414\n",
      "Epoch 3/10\n",
      "\n",
      "Epoch 00003: LearningRateScheduler setting learning rate to 0.15.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.9723 - acc: 0.6670 - val_loss: 1.4169 - val_acc: 0.5868\n",
      "Epoch 4/10\n",
      "\n",
      "Epoch 00004: LearningRateScheduler setting learning rate to 0.2.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.9307 - acc: 0.6879 - val_loss: 1.5187 - val_acc: 0.6040\n",
      "Epoch 5/10\n",
      "\n",
      "Epoch 00005: LearningRateScheduler setting learning rate to 0.25.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.9170 - acc: 0.6971 - val_loss: 1.8209 - val_acc: 0.5569\n",
      "Epoch 6/10\n",
      "\n",
      "Epoch 00006: LearningRateScheduler setting learning rate to 0.3.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.9329 - acc: 0.6972 - val_loss: 2.8204 - val_acc: 0.3688\n",
      "Epoch 7/10\n",
      "\n",
      "Epoch 00007: LearningRateScheduler setting learning rate to 0.35.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.8703 - acc: 0.7173 - val_loss: 1.8094 - val_acc: 0.5186\n",
      "Epoch 8/10\n",
      "\n",
      "Epoch 00008: LearningRateScheduler setting learning rate to 0.4.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.8644 - acc: 0.7246 - val_loss: 1.9543 - val_acc: 0.4545\n",
      "Epoch 9/10\n",
      "\n",
      "Epoch 00009: LearningRateScheduler setting learning rate to 0.45.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.8631 - acc: 0.7262 - val_loss: 2.3720 - val_acc: 0.4789\n",
      "Epoch 10/10\n",
      "\n",
      "Epoch 00010: LearningRateScheduler setting learning rate to 0.5.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.9120 - acc: 0.7224 - val_loss: 1.6642 - val_acc: 0.5663\n"
     ]
    }
   ],
   "source": [
    "hist = M.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=64, epochs=10, callbacks=[\n",
    "    keras.callbacks.LearningRateScheduler(lambda epoch,lr: 0.5*((epoch+1)/10), verbose=1),\n",
    "    keras.callbacks.EarlyStopping(monitor='loss', patience=2, verbose=1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_lr = 0.5*((np.argmin(hist.history['loss'])+1)/10)\n",
    "max_lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cosine annealing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/60\n",
      "\n",
      "Epoch 00001: LearningRateScheduler setting learning rate to 0.45.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.7281 - acc: 0.7688 - val_loss: 0.9205 - val_acc: 0.7283\n",
      "Epoch 2/60\n",
      "\n",
      "Epoch 00002: LearningRateScheduler setting learning rate to 0.40478357377724805.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.7141 - acc: 0.7795 - val_loss: 0.7832 - val_acc: 0.7557\n",
      "Epoch 3/60\n",
      "\n",
      "Epoch 00003: LearningRateScheduler setting learning rate to 0.28125000000000006.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.5420 - acc: 0.8247 - val_loss: 0.9227 - val_acc: 0.7191\n",
      "Epoch 4/60\n",
      "\n",
      "Epoch 00004: LearningRateScheduler setting learning rate to 0.11250000000000002.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3892 - acc: 0.8707 - val_loss: 0.6410 - val_acc: 0.8035\n",
      "Epoch 5/60\n",
      "\n",
      "Epoch 00005: LearningRateScheduler setting learning rate to 0.45.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.6943 - acc: 0.7847 - val_loss: 1.8059 - val_acc: 0.6085\n",
      "Epoch 6/60\n",
      "\n",
      "Epoch 00006: LearningRateScheduler setting learning rate to 0.4415381703613655.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.6139 - acc: 0.8072 - val_loss: 1.1650 - val_acc: 0.6643\n",
      "Epoch 7/60\n",
      "\n",
      "Epoch 00007: LearningRateScheduler setting learning rate to 0.41657699291706646.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.5498 - acc: 0.8259 - val_loss: 1.4670 - val_acc: 0.6273\n",
      "Epoch 8/60\n",
      "\n",
      "Epoch 00008: LearningRateScheduler setting learning rate to 0.37636812533296005.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.5055 - acc: 0.8414 - val_loss: 1.4512 - val_acc: 0.6589\n",
      "Epoch 9/60\n",
      "\n",
      "Epoch 00009: LearningRateScheduler setting learning rate to 0.3229278081273226.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.4470 - acc: 0.8577 - val_loss: 1.0549 - val_acc: 0.7005\n",
      "Epoch 10/60\n",
      "\n",
      "Epoch 00010: LearningRateScheduler setting learning rate to 0.2589357619521759.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.3704 - acc: 0.8802 - val_loss: 0.7968 - val_acc: 0.7894\n",
      "Epoch 11/60\n",
      "\n",
      "Epoch 00011: LearningRateScheduler setting learning rate to 0.18760081521025612.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.2999 - acc: 0.9036 - val_loss: 0.7514 - val_acc: 0.8025\n",
      "Epoch 12/60\n",
      "\n",
      "Epoch 00012: LearningRateScheduler setting learning rate to 0.11250000000000002.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2236 - acc: 0.9281 - val_loss: 0.8055 - val_acc: 0.7959\n",
      "Epoch 13/60\n",
      "\n",
      "Epoch 00013: LearningRateScheduler setting learning rate to 0.45.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.6358 - acc: 0.8159 - val_loss: 1.8469 - val_acc: 0.5816\n",
      "Epoch 14/60\n",
      "\n",
      "Epoch 00014: LearningRateScheduler setting learning rate to 0.44815113968679227.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.5022 - acc: 0.8471 - val_loss: 1.1270 - val_acc: 0.7437\n",
      "Epoch 15/60\n",
      "\n",
      "Epoch 00015: LearningRateScheduler setting learning rate to 0.44262481524765945.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.4645 - acc: 0.8575 - val_loss: 0.9729 - val_acc: 0.7573\n",
      "Epoch 16/60\n",
      "\n",
      "Epoch 00016: LearningRateScheduler setting learning rate to 0.4334815742496143.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.4497 - acc: 0.8628 - val_loss: 1.0597 - val_acc: 0.7311\n",
      "Epoch 17/60\n",
      "\n",
      "Epoch 00017: LearningRateScheduler setting learning rate to 0.4208215919543778.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.4324 - acc: 0.8689 - val_loss: 0.9614 - val_acc: 0.7374\n",
      "Epoch 18/60\n",
      "\n",
      "Epoch 00018: LearningRateScheduler setting learning rate to 0.40478357377724805.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.3995 - acc: 0.8796 - val_loss: 1.2097 - val_acc: 0.7209\n",
      "Epoch 19/60\n",
      "\n",
      "Epoch 00019: LearningRateScheduler setting learning rate to 0.3855432356015448.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3930 - acc: 0.8817 - val_loss: 1.4809 - val_acc: 0.6637\n",
      "Epoch 20/60\n",
      "\n",
      "Epoch 00020: LearningRateScheduler setting learning rate to 0.36331137859862056.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3518 - acc: 0.8932 - val_loss: 1.1260 - val_acc: 0.7607\n",
      "Epoch 21/60\n",
      "\n",
      "Epoch 00021: LearningRateScheduler setting learning rate to 0.33833157964611466.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3304 - acc: 0.9000 - val_loss: 0.9589 - val_acc: 0.7830\n",
      "Epoch 22/60\n",
      "\n",
      "Epoch 00022: LearningRateScheduler setting learning rate to 0.3108775226487097.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2978 - acc: 0.9105 - val_loss: 1.0958 - val_acc: 0.7597\n",
      "Epoch 23/60\n",
      "\n",
      "Epoch 00023: LearningRateScheduler setting learning rate to 0.28125000000000006.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2678 - acc: 0.9206 - val_loss: 1.0064 - val_acc: 0.7669\n",
      "Epoch 24/60\n",
      "\n",
      "Epoch 00024: LearningRateScheduler setting learning rate to 0.24977361703808265.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2403 - acc: 0.9289 - val_loss: 1.1007 - val_acc: 0.7870\n",
      "Epoch 25/60\n",
      "\n",
      "Epoch 00025: LearningRateScheduler setting learning rate to 0.2167932356015448.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2083 - acc: 0.9370 - val_loss: 1.0200 - val_acc: 0.7808\n",
      "Epoch 26/60\n",
      "\n",
      "Epoch 00026: LearningRateScheduler setting learning rate to 0.18267019565099374.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1668 - acc: 0.9496 - val_loss: 1.0962 - val_acc: 0.7829\n",
      "Epoch 27/60\n",
      "\n",
      "Epoch 00027: LearningRateScheduler setting learning rate to 0.14777835635283304.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1483 - acc: 0.9559 - val_loss: 1.0725 - val_acc: 0.8009\n",
      "Epoch 28/60\n",
      "\n",
      "Epoch 00028: LearningRateScheduler setting learning rate to 0.11250000000000002.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1108 - acc: 0.9670 - val_loss: 1.1987 - val_acc: 0.7962\n",
      "Epoch 29/60\n",
      "\n",
      "Epoch 00029: LearningRateScheduler setting learning rate to 0.45.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 1.0412 - acc: 0.6856 - val_loss: 1.7342 - val_acc: 0.5031\n",
      "Epoch 30/60\n",
      "\n",
      "Epoch 00030: LearningRateScheduler setting learning rate to 0.44956682117023034.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.7301 - acc: 0.7734 - val_loss: 1.3252 - val_acc: 0.5849\n",
      "Epoch 31/60\n",
      "\n",
      "Epoch 00031: LearningRateScheduler setting learning rate to 0.44826839664476464.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.5658 - acc: 0.8210 - val_loss: 1.1781 - val_acc: 0.7019\n",
      "Epoch 32/60\n",
      "\n",
      "Epoch 00032: LearningRateScheduler setting learning rate to 0.4461080594607376.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.6451 - acc: 0.8052 - val_loss: 1.1798 - val_acc: 0.6529\n",
      "Epoch 33/60\n",
      "\n",
      "Epoch 00033: LearningRateScheduler setting learning rate to 0.4430913551727169.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.4659 - acc: 0.8543 - val_loss: 1.1731 - val_acc: 0.7436\n",
      "Epoch 34/60\n",
      "\n",
      "Epoch 00034: LearningRateScheduler setting learning rate to 0.439226027617344.\n",
      "50000/50000 [==============================] - 56s 1ms/step - loss: 0.4519 - acc: 0.8605 - val_loss: 1.0918 - val_acc: 0.7215\n",
      "Epoch 35/60\n",
      "\n",
      "Epoch 00035: LearningRateScheduler setting learning rate to 0.4345219990350165.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.4054 - acc: 0.8721 - val_loss: 0.9731 - val_acc: 0.7646\n",
      "Epoch 36/60\n",
      "\n",
      "Epoch 00036: LearningRateScheduler setting learning rate to 0.42899134459963967.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.4095 - acc: 0.8757 - val_loss: 1.1912 - val_acc: 0.7528\n",
      "Epoch 37/60\n",
      "\n",
      "Epoch 00037: LearningRateScheduler setting learning rate to 0.4226482614218278.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.4031 - acc: 0.8787 - val_loss: 1.1160 - val_acc: 0.7477\n",
      "Epoch 38/60\n",
      "\n",
      "Epoch 00038: LearningRateScheduler setting learning rate to 0.41550903210512535.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3714 - acc: 0.8874 - val_loss: 1.2731 - val_acc: 0.7366\n",
      "Epoch 39/60\n",
      "\n",
      "Epoch 00039: LearningRateScheduler setting learning rate to 0.40759198294879645.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3531 - acc: 0.8924 - val_loss: 1.5498 - val_acc: 0.6797\n",
      "Epoch 40/60\n",
      "\n",
      "Epoch 00040: LearningRateScheduler setting learning rate to 0.39891743690447845.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.3358 - acc: 0.8994 - val_loss: 1.2462 - val_acc: 0.7519\n",
      "Epoch 41/60\n",
      "\n",
      "Epoch 00041: LearningRateScheduler setting learning rate to 0.38950766140745574.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3240 - acc: 0.9034 - val_loss: 1.4744 - val_acc: 0.7134\n",
      "Epoch 42/60\n",
      "\n",
      "Epoch 00042: LearningRateScheduler setting learning rate to 0.3793868112164733.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.3100 - acc: 0.9060 - val_loss: 1.1497 - val_acc: 0.7672\n",
      "Epoch 43/60\n",
      "\n",
      "Epoch 00043: LearningRateScheduler setting learning rate to 0.36858086640881693.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2939 - acc: 0.9111 - val_loss: 1.4644 - val_acc: 0.7493\n",
      "Epoch 44/60\n",
      "\n",
      "Epoch 00044: LearningRateScheduler setting learning rate to 0.357117565689828.\n",
      "50000/50000 [==============================] - 56s 1ms/step - loss: 0.3044 - acc: 0.9088 - val_loss: 1.1858 - val_acc: 0.7707\n",
      "Epoch 45/60\n",
      "\n",
      "Epoch 00045: LearningRateScheduler setting learning rate to 0.34502633518804426.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.2655 - acc: 0.9205 - val_loss: 1.3388 - val_acc: 0.7483\n",
      "Epoch 46/60\n",
      "\n",
      "Epoch 00046: LearningRateScheduler setting learning rate to 0.33233821291875004.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.2544 - acc: 0.9241 - val_loss: 1.2766 - val_acc: 0.7527\n",
      "Epoch 47/60\n",
      "\n",
      "Epoch 00047: LearningRateScheduler setting learning rate to 0.3190857691098362.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.2392 - acc: 0.9289 - val_loss: 1.3794 - val_acc: 0.7625\n",
      "Epoch 48/60\n",
      "\n",
      "Epoch 00048: LearningRateScheduler setting learning rate to 0.30530302259449243.\n",
      "50000/50000 [==============================] - 53s 1ms/step - loss: 0.2077 - acc: 0.9400 - val_loss: 1.2251 - val_acc: 0.7605\n",
      "Epoch 49/60\n",
      "\n",
      "Epoch 00049: LearningRateScheduler setting learning rate to 0.2910253534853498.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.2052 - acc: 0.9396 - val_loss: 1.2472 - val_acc: 0.7756\n",
      "Epoch 50/60\n",
      "\n",
      "Epoch 00050: LearningRateScheduler setting learning rate to 0.2762894123542399.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1951 - acc: 0.9422 - val_loss: 1.3237 - val_acc: 0.7644\n",
      "Epoch 51/60\n",
      "\n",
      "Epoch 00051: LearningRateScheduler setting learning rate to 0.2611330261507016.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.1770 - acc: 0.9480 - val_loss: 1.3176 - val_acc: 0.7764\n",
      "Epoch 52/60\n",
      "\n",
      "Epoch 00052: LearningRateScheduler setting learning rate to 0.245595101100745.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1787 - acc: 0.9484 - val_loss: 1.3377 - val_acc: 0.7675\n",
      "Epoch 53/60\n",
      "\n",
      "Epoch 00053: LearningRateScheduler setting learning rate to 0.22971552283512686.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1601 - acc: 0.9529 - val_loss: 1.4466 - val_acc: 0.7762\n",
      "Epoch 54/60\n",
      "\n",
      "Epoch 00054: LearningRateScheduler setting learning rate to 0.2135350540035084.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1482 - acc: 0.9560 - val_loss: 1.4350 - val_acc: 0.7822\n",
      "Epoch 55/60\n",
      "\n",
      "Epoch 00055: LearningRateScheduler setting learning rate to 0.19709522963731818.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1318 - acc: 0.9613 - val_loss: 1.5851 - val_acc: 0.7617\n",
      "Epoch 56/60\n",
      "\n",
      "Epoch 00056: LearningRateScheduler setting learning rate to 0.1804382505299228.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.1169 - acc: 0.9659 - val_loss: 1.4096 - val_acc: 0.7898\n",
      "Epoch 57/60\n",
      "\n",
      "Epoch 00057: LearningRateScheduler setting learning rate to 0.16360687490779463.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.1065 - acc: 0.9689 - val_loss: 1.4992 - val_acc: 0.7816\n",
      "Epoch 58/60\n",
      "\n",
      "Epoch 00058: LearningRateScheduler setting learning rate to 0.1466443086707584.\n",
      "50000/50000 [==============================] - 55s 1ms/step - loss: 0.0998 - acc: 0.9712 - val_loss: 1.5281 - val_acc: 0.7776\n",
      "Epoch 59/60\n",
      "\n",
      "Epoch 00059: LearningRateScheduler setting learning rate to 0.12959409448306555.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.0815 - acc: 0.9760 - val_loss: 1.5702 - val_acc: 0.7822\n",
      "Epoch 60/60\n",
      "\n",
      "Epoch 00060: LearningRateScheduler setting learning rate to 0.11250000000000002.\n",
      "50000/50000 [==============================] - 54s 1ms/step - loss: 0.0732 - acc: 0.9784 - val_loss: 1.5577 - val_acc: 0.7844\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3fe858cdd8>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def cosine_annealing(i, lr_min, lr_max):\n",
    "    i = i + np.power(2,2)\n",
    "    log_i = np.log2(i)\n",
    "    t_min,t_max = np.power(2,np.floor(log_i)), np.power(2,np.floor(log_i)+1)-1\n",
    "    return lr_min + np.cos(((i-t_min)/(t_max-t_min))*(np.pi/2))*(lr_max-lr_min)\n",
    "M.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=64, epochs=60, callbacks=[\n",
    "    keras.callbacks.LearningRateScheduler(\n",
    "        lambda epoch,lr: cosine_annealing(epoch,max_lr*0.25,max_lr), verbose=1)\n",
    "])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}