GANs Simplified.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.datasets import mnist\n",
    "from keras.optimizers import Adam\n",
    "from keras import initializers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To make sure that we can reproduce the experiment and get the same results\n",
    "np.random.seed(10)\n",
    "# The dimension of our random noise vector.\n",
    "random_dim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_minst_data():\n",
    "    # load the data\n",
    "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "    # normalize our inputs to be in the range[-1, 1] \n",
    "    x_train = (x_train.astype(np.float32) - 127.5)/127.5\n",
    "    # convert x_train with a shape of (60000, 28, 28) to (60000, 784) so we have\n",
    "    # 784 columns per row\n",
    "    x_train = x_train.reshape(60000, 784)\n",
    "    return (x_train, y_train, x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimizer():\n",
    "    return Adam(lr=0.0002, beta_1=0.5) \n",
    "\n",
    "def get_generator(optimizer):\n",
    "    generator = Sequential()\n",
    "    generator.add(Dense(256, input_dim=random_dim, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "    generator.add(LeakyReLU(0.2))\n",
    "\n",
    "    generator.add(Dense(512))\n",
    "    generator.add(LeakyReLU(0.2))\n",
    "\n",
    "    generator.add(Dense(1024))\n",
    "    generator.add(LeakyReLU(0.2))\n",
    "\n",
    "    generator.add(Dense(784, activation='tanh'))\n",
    "    generator.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "    return generator\n",
    "\n",
    "def get_discriminator(optimizer):\n",
    "    discriminator = Sequential()\n",
    "    discriminator.add(Dense(1024, input_dim=784, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "    discriminator.add(LeakyReLU(0.2))\n",
    "    discriminator.add(Dropout(0.3))\n",
    "\n",
    "    discriminator.add(Dense(512))\n",
    "    discriminator.add(LeakyReLU(0.2))\n",
    "    discriminator.add(Dropout(0.3))\n",
    "\n",
    "    discriminator.add(Dense(256))\n",
    "    discriminator.add(LeakyReLU(0.2))\n",
    "    discriminator.add(Dropout(0.3))\n",
    "\n",
    "    discriminator.add(Dense(1, activation='sigmoid'))\n",
    "    discriminator.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "    return discriminator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gan_network(discriminator, random_dim, generator, optimizer):\n",
    "    # We initially set trainable to False since we only want to train either the \n",
    "    # generator or discriminator at a time\n",
    "    discriminator.trainable = False\n",
    "    # gan input (noise) will be 100-dimensional vectors\n",
    "    gan_input = Input(shape=(random_dim,))\n",
    "    # the output of the generator (an image)\n",
    "    x = generator(gan_input)\n",
    "    # get the output of the discriminator (probability if the image is real or not)\n",
    "    gan_output = discriminator(x)\n",
    "    gan = Model(inputs=gan_input, outputs=gan_output)\n",
    "    gan.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "    return gan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_generated_images(epoch, generator, examples=100, dim=(10, 10), figsize=(10, 10)):\n",
    "    noise = np.random.normal(0, 1, size=[examples, random_dim])\n",
    "    generated_images = generator.predict(noise)\n",
    "    generated_images = generated_images.reshape(examples, 28, 28)\n",
    "\n",
    "    plt.figure(figsize=figsize)\n",
    "    for i in range(generated_images.shape[0]):\n",
    "        plt.subplot(dim[0], dim[1], i+1)\n",
    "        plt.imshow(generated_images[i], interpolation='nearest', cmap='gray_r')\n",
    "        plt.axis('off')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('gan_generated_image_epoch_%d.png' % epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 32s 3us/step\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'xrange' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-0c757600abcc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m400\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-0c757600abcc>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(epochs, batch_size)\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mgan\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_gan_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiscriminator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m         \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'-'\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Epoch %d'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'-'\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'xrange' is not defined"
     ]
    }
   ],
   "source": [
    "def train(epochs=1, batch_size=128):\n",
    "    # Get the training and testing data\n",
    "    x_train, y_train, x_test, y_test = load_minst_data()\n",
    "    # Split the training data into batches of size 128\n",
    "    batch_count = x_train.shape[0] / batch_size\n",
    "\n",
    "    # Build our GAN netowrk\n",
    "    adam = get_optimizer()\n",
    "    generator = get_generator(adam)\n",
    "    discriminator = get_discriminator(adam)\n",
    "    gan = get_gan_network(discriminator, random_dim, generator, adam)\n",
    "\n",
    "    for e in xrange(1, epochs+1):\n",
    "        print ('-'*15, 'Epoch %d' % e, '-'*15)\n",
    "        for _ in tqdm(xrange(batch_count)):\n",
    "            # Get a random set of input noise and images\n",
    "            noise = np.random.normal(0, 1, size=[batch_size, random_dim])\n",
    "            image_batch = x_train[np.random.randint(0, x_train.shape[0], size=batch_size)]\n",
    "\n",
    "            # Generate fake MNIST images\n",
    "            generated_images = generator.predict(noise)\n",
    "            X = np.concatenate([image_batch, generated_images])\n",
    "\n",
    "            # Labels for generated and real data\n",
    "            y_dis = np.zeros(2*batch_size)\n",
    "            # One-sided label smoothing\n",
    "            y_dis[:batch_size] = 0.9\n",
    "\n",
    "            # Train discriminator\n",
    "            discriminator.trainable = True\n",
    "            discriminator.train_on_batch(X, y_dis)\n",
    "\n",
    "            # Train generator\n",
    "            noise = np.random.normal(0, 1, size=[batch_size, random_dim])\n",
    "            y_gen = np.ones(batch_size)\n",
    "            discriminator.trainable = False\n",
    "            gan.train_on_batch(noise, y_gen)\n",
    "\n",
    "        if e == 1 or e % 20 == 0:\n",
    "            plot_generated_images(e, generator)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    train(400, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting mnist/train-images-idx3-ubyte.gz\n",
      "Extracting mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting mnist/t10k-labels-idx1-ubyte.gz\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_5 (Conv2D)            (None, 14, 14, 64)        1664      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_11 (LeakyReLU)   (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 7, 7, 128)         204928    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)   (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 4, 4, 256)         819456    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, 4, 4, 512)         3277312   \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)   (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 1)                 8193      \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 4,311,553\n",
      "Trainable params: 4,311,553\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 12544)             1266944   \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 12544)             50176     \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 12544)             0         \n",
      "_________________________________________________________________\n",
      "reshape_2 (Reshape)          (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "up_sampling2d_3 (UpSampling2 (None, 14, 14, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_5 (Conv2DTr (None, 14, 14, 128)       819328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "up_sampling2d_4 (UpSampling2 (None, 28, 28, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_6 (Conv2DTr (None, 28, 28, 64)        204864    \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_7 (Conv2DTr (None, 28, 28, 32)        51232     \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 28, 28, 32)        128       \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 28, 28, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_8 (Conv2DTr (None, 28, 28, 1)         801       \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 28, 28, 1)         0         \n",
      "=================================================================\n",
      "Total params: 2,394,241\n",
      "Trainable params: 2,368,705\n",
      "Non-trainable params: 25,536\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-9abc4f4de10d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    200\u001b[0m     \u001b[0mmnist_dcgan\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMNIST_DCGAN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m     \u001b[0mtimer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mElapsedTimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 202\u001b[0;31m     \u001b[0mmnist_dcgan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_interval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    203\u001b[0m     \u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melapsed_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0mmnist_dcgan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfake\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-9abc4f4de10d>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_steps, batch_size, save_interval)\u001b[0m\n\u001b[1;32m    163\u001b[0m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m             \u001b[0mnoise\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m             \u001b[0ma_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madversarial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnoise\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m             \u001b[0mlog_mesg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"%d: [D loss: %f, acc: %f]\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m             \u001b[0mlog_mesg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"%s  [A loss: %f, acc: %f]\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlog_mesg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, class_weight, sample_weight)\u001b[0m\n\u001b[1;32m    953\u001b[0m         return self.model.train_on_batch(x, y,\n\u001b[1;32m    954\u001b[0m                                          \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 955\u001b[0;31m                                          class_weight=class_weight)\n\u001b[0m\u001b[1;32m    956\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m     def test_on_batch(self, x, y,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1760\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1761\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1763\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1764\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1316\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1317\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1320\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1322\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1323\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1307\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1409\u001b[0;31m           run_metadata)\n\u001b[0m\u001b[1;32m   1410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "'''\n",
    "DCGAN on MNIST using Keras\n",
    "Author: Rowel Atienza\n",
    "Project: https://github.com/roatienza/Deep-Learning-Experiments\n",
    "Dependencies: tensorflow 1.0 and keras 2.0\n",
    "Usage: python3 dcgan_mnist.py\n",
    "'''\n",
    "\n",
    "import numpy as np\n",
    "import time\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Flatten, Reshape\n",
    "from keras.layers import Conv2D, Conv2DTranspose, UpSampling2D\n",
    "from keras.layers import LeakyReLU, Dropout\n",
    "from keras.layers import BatchNormalization\n",
    "from keras.optimizers import Adam, RMSprop\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class ElapsedTimer(object):\n",
    "    def __init__(self):\n",
    "        self.start_time = time.time()\n",
    "    def elapsed(self,sec):\n",
    "        if sec < 60:\n",
    "            return str(sec) + \" sec\"\n",
    "        elif sec < (60 * 60):\n",
    "            return str(sec / 60) + \" min\"\n",
    "        else:\n",
    "            return str(sec / (60 * 60)) + \" hr\"\n",
    "    def elapsed_time(self):\n",
    "        print(\"Elapsed: %s \" % self.elapsed(time.time() - self.start_time) )\n",
    "\n",
    "class DCGAN(object):\n",
    "    def __init__(self, img_rows=28, img_cols=28, channel=1):\n",
    "\n",
    "        self.img_rows = img_rows\n",
    "        self.img_cols = img_cols\n",
    "        self.channel = channel\n",
    "        self.D = None   # discriminator\n",
    "        self.G = None   # generator\n",
    "        self.AM = None  # adversarial model\n",
    "        self.DM = None  # discriminator model\n",
    "\n",
    "    # (W−F+2P)/S+1\n",
    "    def discriminator(self):\n",
    "        if self.D:\n",
    "            return self.D\n",
    "        self.D = Sequential()\n",
    "        depth = 64\n",
    "        dropout = 0.4\n",
    "        # In: 28 x 28 x 1, depth = 1\n",
    "        # Out: 14 x 14 x 1, depth=64\n",
    "        input_shape = (self.img_rows, self.img_cols, self.channel)\n",
    "        self.D.add(Conv2D(depth*1, 5, strides=2, input_shape=input_shape,\\\n",
    "            padding='same'))\n",
    "        self.D.add(LeakyReLU(alpha=0.2))\n",
    "        self.D.add(Dropout(dropout))\n",
    "\n",
    "        self.D.add(Conv2D(depth*2, 5, strides=2, padding='same'))\n",
    "        self.D.add(LeakyReLU(alpha=0.2))\n",
    "        self.D.add(Dropout(dropout))\n",
    "\n",
    "        self.D.add(Conv2D(depth*4, 5, strides=2, padding='same'))\n",
    "        self.D.add(LeakyReLU(alpha=0.2))\n",
    "        self.D.add(Dropout(dropout))\n",
    "\n",
    "        self.D.add(Conv2D(depth*8, 5, strides=1, padding='same'))\n",
    "        self.D.add(LeakyReLU(alpha=0.2))\n",
    "        self.D.add(Dropout(dropout))\n",
    "\n",
    "        # Out: 1-dim probability\n",
    "        self.D.add(Flatten())\n",
    "        self.D.add(Dense(1))\n",
    "        self.D.add(Activation('sigmoid'))\n",
    "        self.D.summary()\n",
    "        return self.D\n",
    "\n",
    "    def generator(self):\n",
    "        if self.G:\n",
    "            return self.G\n",
    "        self.G = Sequential()\n",
    "        dropout = 0.4\n",
    "        depth = 64+64+64+64\n",
    "        dim = 7\n",
    "        # In: 100\n",
    "        # Out: dim x dim x depth\n",
    "        self.G.add(Dense(dim*dim*depth, input_dim=100))\n",
    "        self.G.add(BatchNormalization(momentum=0.9))\n",
    "        self.G.add(Activation('relu'))\n",
    "        self.G.add(Reshape((dim, dim, depth)))\n",
    "        self.G.add(Dropout(dropout))\n",
    "\n",
    "        # In: dim x dim x depth\n",
    "        # Out: 2*dim x 2*dim x depth/2\n",
    "        self.G.add(UpSampling2D())\n",
    "        self.G.add(Conv2DTranspose(int(depth/2), 5, padding='same'))\n",
    "        self.G.add(BatchNormalization(momentum=0.9))\n",
    "        self.G.add(Activation('relu'))\n",
    "\n",
    "        self.G.add(UpSampling2D())\n",
    "        self.G.add(Conv2DTranspose(int(depth/4), 5, padding='same'))\n",
    "        self.G.add(BatchNormalization(momentum=0.9))\n",
    "        self.G.add(Activation('relu'))\n",
    "\n",
    "        self.G.add(Conv2DTranspose(int(depth/8), 5, padding='same'))\n",
    "        self.G.add(BatchNormalization(momentum=0.9))\n",
    "        self.G.add(Activation('relu'))\n",
    "\n",
    "        # Out: 28 x 28 x 1 grayscale image [0.0,1.0] per pix\n",
    "        self.G.add(Conv2DTranspose(1, 5, padding='same'))\n",
    "        self.G.add(Activation('sigmoid'))\n",
    "        self.G.summary()\n",
    "        return self.G\n",
    "\n",
    "    def discriminator_model(self):\n",
    "        if self.DM:\n",
    "            return self.DM\n",
    "        optimizer = RMSprop(lr=0.0002, decay=6e-8)\n",
    "        self.DM = Sequential()\n",
    "        self.DM.add(self.discriminator())\n",
    "        self.DM.compile(loss='binary_crossentropy', optimizer=optimizer,\\\n",
    "            metrics=['accuracy'])\n",
    "        return self.DM\n",
    "\n",
    "    def adversarial_model(self):\n",
    "        if self.AM:\n",
    "            return self.AM\n",
    "        optimizer = RMSprop(lr=0.0001, decay=3e-8)\n",
    "        self.AM = Sequential()\n",
    "        self.AM.add(self.generator())\n",
    "        self.AM.add(self.discriminator())\n",
    "        self.AM.compile(loss='binary_crossentropy', optimizer=optimizer,\\\n",
    "            metrics=['accuracy'])\n",
    "        return self.AM\n",
    "\n",
    "class MNIST_DCGAN(object):\n",
    "    def __init__(self):\n",
    "        self.img_rows = 28\n",
    "        self.img_cols = 28\n",
    "        self.channel = 1\n",
    "\n",
    "        self.x_train = input_data.read_data_sets(\"mnist\",\\\n",
    "        \tone_hot=True).train.images\n",
    "        self.x_train = self.x_train.reshape(-1, self.img_rows,\\\n",
    "        \tself.img_cols, 1).astype(np.float32)\n",
    "\n",
    "        self.DCGAN = DCGAN()\n",
    "        self.discriminator =  self.DCGAN.discriminator_model()\n",
    "        self.adversarial = self.DCGAN.adversarial_model()\n",
    "        self.generator = self.DCGAN.generator()\n",
    "\n",
    "    def train(self, train_steps=2000, batch_size=256, save_interval=0):\n",
    "        noise_input = None\n",
    "        if save_interval>0:\n",
    "            noise_input = np.random.uniform(-1.0, 1.0, size=[16, 100])\n",
    "        for i in range(train_steps):\n",
    "            images_train = self.x_train[np.random.randint(0,\n",
    "                self.x_train.shape[0], size=batch_size), :, :, :]\n",
    "            noise = np.random.uniform(-1.0, 1.0, size=[batch_size, 100])\n",
    "            images_fake = self.generator.predict(noise)\n",
    "            x = np.concatenate((images_train, images_fake))\n",
    "            y = np.ones([2*batch_size, 1])\n",
    "            y[batch_size:, :] = 0\n",
    "            d_loss = self.discriminator.train_on_batch(x, y)\n",
    "\n",
    "            y = np.ones([batch_size, 1])\n",
    "            noise = np.random.uniform(-1.0, 1.0, size=[batch_size, 100])\n",
    "            a_loss = self.adversarial.train_on_batch(noise, y)\n",
    "            log_mesg = \"%d: [D loss: %f, acc: %f]\" % (i, d_loss[0], d_loss[1])\n",
    "            log_mesg = \"%s  [A loss: %f, acc: %f]\" % (log_mesg, a_loss[0], a_loss[1])\n",
    "            print(log_mesg)\n",
    "            if save_interval>0:\n",
    "                if (i+1)%save_interval==0:\n",
    "                    self.plot_images(save2file=True, samples=noise_input.shape[0],\\\n",
    "                        noise=noise_input, step=(i+1))\n",
    "\n",
    "    def plot_images(self, save2file=False, fake=True, samples=16, noise=None, step=0):\n",
    "        filename = 'mnist.png'\n",
    "        if fake:\n",
    "            if noise is None:\n",
    "                noise = np.random.uniform(-1.0, 1.0, size=[samples, 100])\n",
    "            else:\n",
    "                filename = \"mnist_%d.png\" % step\n",
    "            images = self.generator.predict(noise)\n",
    "        else:\n",
    "            i = np.random.randint(0, self.x_train.shape[0], samples)\n",
    "            images = self.x_train[i, :, :, :]\n",
    "\n",
    "        plt.figure(figsize=(10,10))\n",
    "        for i in range(images.shape[0]):\n",
    "            plt.subplot(4, 4, i+1)\n",
    "            image = images[i, :, :, :]\n",
    "            image = np.reshape(image, [self.img_rows, self.img_cols])\n",
    "            plt.imshow(image, cmap='gray')\n",
    "            plt.axis('off')\n",
    "        plt.tight_layout()\n",
    "        if save2file:\n",
    "            plt.savefig(filename)\n",
    "            plt.close('all')\n",
    "        else:\n",
    "            plt.show()\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    mnist_dcgan = MNIST_DCGAN()\n",
    "    timer = ElapsedTimer()\n",
    "    mnist_dcgan.train(train_steps=10000, batch_size=256, save_interval=500)\n",
    "    timer.elapsed_time()\n",
    "    mnist_dcgan.plot_images(fake=True)\n",
    "    mnist_dcgan.plot_images(fake=False, save2file=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "  0%|          | 0/468 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 200\n",
      "Batch size: 128\n",
      "Batches per epoch: 468.75\n",
      "--------------- Epoch 1 ---------------\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers.core import Reshape, Dense, Dropout, Flatten\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.convolutional import Convolution2D, UpSampling2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.datasets import mnist\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "from keras import initializers\n",
    "\n",
    "K.set_image_dim_ordering('th')\n",
    "\n",
    "# Deterministic output.\n",
    "# Tired of seeing the same results every time? Remove the line below.\n",
    "np.random.seed(1000)\n",
    "\n",
    "# The results are a little better when the dimensionality of the random vector is only 10.\n",
    "# The dimensionality has been left at 100 for consistency with other GAN implementations.\n",
    "randomDim = 100\n",
    "\n",
    "# Load MNIST data\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5)/127.5\n",
    "X_train = X_train.reshape(60000, 784)\n",
    "\n",
    "# Optimizer\n",
    "adam = Adam(lr=0.0002, beta_1=0.5)\n",
    "\n",
    "generator = Sequential()\n",
    "generator.add(Dense(256, input_dim=randomDim, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(512))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(1024))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(784, activation='tanh'))\n",
    "generator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "discriminator = Sequential()\n",
    "discriminator.add(Dense(1024, input_dim=784, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(512))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(256))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(1, activation='sigmoid'))\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "# Combined network\n",
    "discriminator.trainable = False\n",
    "ganInput = Input(shape=(randomDim,))\n",
    "x = generator(ganInput)\n",
    "ganOutput = discriminator(x)\n",
    "gan = Model(inputs=ganInput, outputs=ganOutput)\n",
    "gan.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "dLosses = []\n",
    "gLosses = []\n",
    "\n",
    "# Plot the loss from each batch\n",
    "def plotLoss(epoch):\n",
    "    plt.figure(figsize=(10, 8))\n",
    "    plt.plot(dLosses, label='Discriminitive loss')\n",
    "    plt.plot(gLosses, label='Generative loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    plt.savefig('images/gan_loss_epoch_%d.png' % epoch)\n",
    "\n",
    "# Create a wall of generated MNIST images\n",
    "def plotGeneratedImages(epoch, examples=100, dim=(10, 10), figsize=(10, 10)):\n",
    "    noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "    generatedImages = generator.predict(noise)\n",
    "    generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "\n",
    "    plt.figure(figsize=figsize)\n",
    "    for i in range(generatedImages.shape[0]):\n",
    "        plt.subplot(dim[0], dim[1], i+1)\n",
    "        plt.imshow(generatedImages[i], interpolation='nearest', cmap='gray_r')\n",
    "        plt.axis('off')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('images/gan_generated_image_epoch_%d.png' % epoch)\n",
    "\n",
    "# Save the generator and discriminator networks (and weights) for later use\n",
    "def saveModels(epoch):\n",
    "    generator.save('models/gan_generator_epoch_%d.h5' % epoch)\n",
    "    discriminator.save('models/gan_discriminator_epoch_%d.h5' % epoch)\n",
    "\n",
    "def train(epochs=1, batchSize=128):\n",
    "    batchCount = X_train.shape[0] / batchSize\n",
    "    print ('Epochs:', epochs)\n",
    "    print ('Batch size:', batchSize)\n",
    "    print ('Batches per epoch:', batchCount)\n",
    "\n",
    "    for e in range(1, epochs+1):\n",
    "        print ('-'*15, 'Epoch %d' % e, '-'*15)\n",
    "        for _ in tqdm(range(int(batchCount))):\n",
    "            # Get a random set of input noise and images\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            imageBatch = X_train[np.random.randint(0, X_train.shape[0], size=batchSize)]\n",
    "\n",
    "            # Generate fake MNIST images\n",
    "            generatedImages = generator.predict(noise)\n",
    "            # print np.shape(imageBatch), np.shape(generatedImages)\n",
    "            X = np.concatenate([imageBatch, generatedImages])\n",
    "\n",
    "            # Labels for generated and real data\n",
    "            yDis = np.zeros(2*batchSize)\n",
    "            # One-sided label smoothing\n",
    "            yDis[:batchSize] = 0.9\n",
    "\n",
    "            # Train discriminator\n",
    "            discriminator.trainable = True\n",
    "            dloss = discriminator.train_on_batch(X, yDis)\n",
    "\n",
    "            # Train generator\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            yGen = np.ones(batchSize)\n",
    "            discriminator.trainable = False\n",
    "            gloss = gan.train_on_batch(noise, yGen)\n",
    "\n",
    "        # Store loss of most recent batch from this epoch\n",
    "        dLosses.append(dloss)\n",
    "        gLosses.append(gloss)\n",
    "\n",
    "        if e == 1 or e % 20 == 0:\n",
    "            plotGeneratedImages(e)\n",
    "            saveModels(e)\n",
    "\n",
    "    # Plot losses from every epoch\n",
    "    plotLoss(e)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    train(200, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}