text2img_s1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Resumable training stackgan_stage_I_imp.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fBRj1Lxsle4H",
        "colab_type": "text"
      },
      "source": [
        "# Stage I GAN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YY7k9WAWXwNt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## this is file of stage 1 of stackgan.\n",
        "## pre-requirements : download birds.zip file\n",
        "## resumable training is added to the code\n",
        "## go-to cell number 5 and change variable as you need. "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "psC9UHjzliA1",
        "colab_type": "text"
      },
      "source": [
        "# Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IOaGz7wUmPy5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "###### javascript function\n",
        "## connect-push ---- paste this in consle\n",
        "\n",
        "function ConnectButton(){\n",
        "    console.log(\"Connect pushed\"); \n",
        "    document.querySelector(\"#top-toolbar > colab-connect-button\").shadowRoot.querySelector(\"#connect\").click() \n",
        "}\n",
        "setInterval(ConnectButton,60000);"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlD1b7xkbrrV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6-iScGs1WcNl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "###### birds dataset for dis.\n",
        "\n",
        "#### download birds.zip form given link\n",
        "## mega link ----  https://mega.nz/file/T4dlRKoL#GrqNQm-c2F8iGyQ5HfvlbzIhI7yJJPFnPdDWyavlAv8\n",
        "\n",
        "!cp /content/drive/'My Drive'/GAN/birds.zip /content/    ### change first arg to birds.zip file in drive\n",
        "!unzip birds.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FmT-KyDR2wLD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#### change variable as per traning of model\n",
        "\n",
        "previousmodel = '3005'  ### previous model (set as date)\n",
        "currentmodel = '1106'   ### current model (set as date)\n",
        "modelpath = 'date3005/'   ### drive directory to fetch previous trained model\n",
        "weightpath = 'date1106/'   ### create directory in drive to store models\n",
        "createdir = 'date1106/models/'   ### store models\n",
        "dirpath = '/content/drive/\"My Drive\"/GAN/'   ### drive path to store current models and its weights\n",
        "endpath = 'models/*'   ### for copy command\n",
        "\n",
        "\n",
        "SET_EPOCHS = 2   ### number of epochs (set default to 300)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExXt0CPk3YPD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## load model from drive\n",
        "!mkdir -p /content/models/\n",
        "###### copy model \n",
        "!cp -r $dirpath$modelpath$endpath /content/models/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ml58N7F-jnLh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import time\n",
        "\n",
        "import PIL\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from PIL import Image\n",
        "from keras import Input, Model\n",
        "from keras import backend as K\n",
        "from keras.callbacks import TensorBoard\n",
        "from keras.layers import Dense, LeakyReLU, BatchNormalization, ReLU, Reshape, UpSampling2D, Conv2D, Activation, \\\n",
        "    concatenate, Flatten, Lambda, Concatenate\n",
        "from keras.optimizers import Adam\n",
        "from matplotlib import pyplot as plt\n",
        "from keras.models import load_model\n",
        "from keras.models import model_from_json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XZybHodapH3H",
        "colab_type": "text"
      },
      "source": [
        "# Loading of Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eMvLOjXalnKd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_class_ids(class_info_file_path):\n",
        "    \"\"\"\n",
        "    Load class ids from class_info.pickle file\n",
        "    \"\"\"\n",
        "    with open(class_info_file_path, 'rb') as f:\n",
        "        class_ids = pickle.load(f, encoding='latin1')\n",
        "        return class_ids"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifdCVSGco5pD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_embeddings(embeddings_file_path):\n",
        "    \"\"\"\n",
        "    Load embeddings\n",
        "    \"\"\"\n",
        "    with open(embeddings_file_path, 'rb') as f:\n",
        "        embeddings = pickle.load(f, encoding='latin1')\n",
        "        embeddings = np.array(embeddings)\n",
        "        print('embeddings: ', embeddings.shape)\n",
        "    return embeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkwKbQgvo7pG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_filenames(filenames_file_path):\n",
        "    \"\"\"\n",
        "    Load filenames.pickle file and return a list of all file names\n",
        "    \"\"\"\n",
        "    with open(filenames_file_path, 'rb') as f:\n",
        "        filenames = pickle.load(f, encoding='latin1')\n",
        "    return filenames"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YxlUAsjuo-i5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_bounding_boxes(dataset_dir):\n",
        "    \"\"\"\n",
        "    Load bounding boxes and return a dictionary of file names and corresponding bounding boxes\n",
        "    \"\"\"\n",
        "    # Paths\n",
        "    bounding_boxes_path = os.path.join(dataset_dir, 'bounding_boxes.txt')\n",
        "    file_paths_path = os.path.join(dataset_dir, 'images.txt')\n",
        "\n",
        "    # Read bounding_boxes.txt and images.txt file\n",
        "    df_bounding_boxes = pd.read_csv(bounding_boxes_path,\n",
        "                                    delim_whitespace=True, header=None).astype(int)\n",
        "    df_file_names = pd.read_csv(file_paths_path, delim_whitespace=True, header=None)\n",
        "\n",
        "    # Create a list of file names\n",
        "    file_names = df_file_names[1].tolist()\n",
        "\n",
        "    # Create a dictionary of file_names and bounding boxes\n",
        "    filename_boundingbox_dict = {img_file[:-4]: [] for img_file in file_names[:2]}\n",
        "\n",
        "    # Assign a bounding box to the corresponding image\n",
        "    for i in range(0, len(file_names)):\n",
        "        # Get the bounding box\n",
        "        bounding_box = df_bounding_boxes.iloc[i][1:].tolist()\n",
        "        key = file_names[i][:-4]\n",
        "        filename_boundingbox_dict[key] = bounding_box\n",
        "\n",
        "    return filename_boundingbox_dict\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQf-MsWtpAyk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_img(img_path, bbox, image_size):\n",
        "    \"\"\"\n",
        "    Load and resize image\n",
        "    \"\"\"\n",
        "    img = Image.open(img_path).convert('RGB')\n",
        "    width, height = img.size\n",
        "    if bbox is not None:\n",
        "        R = int(np.maximum(bbox[2], bbox[3]) * 0.75)\n",
        "        center_x = int((2 * bbox[0] + bbox[2]) / 2)\n",
        "        center_y = int((2 * bbox[1] + bbox[3]) / 2)\n",
        "        y1 = np.maximum(0, center_y - R)\n",
        "        y2 = np.minimum(height, center_y + R)\n",
        "        x1 = np.maximum(0, center_x - R)\n",
        "        x2 = np.minimum(width, center_x + R)\n",
        "        img = img.crop([x1, y1, x2, y2])\n",
        "    img = img.resize(image_size, PIL.Image.BILINEAR)\n",
        "    return img"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gk-4oF83Su2v",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BEtbSBB8pCfo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_dataset(filenames_file_path, class_info_file_path, cub_dataset_dir, embeddings_file_path, image_size):\n",
        "    \"\"\"\n",
        "    Load dataset\n",
        "    \"\"\"\n",
        "    filenames = load_filenames(filenames_file_path)\n",
        "    class_ids = load_class_ids(class_info_file_path)\n",
        "    bounding_boxes = load_bounding_boxes(cub_dataset_dir)\n",
        "    all_embeddings = load_embeddings(embeddings_file_path)\n",
        "\n",
        "    X, y, embeddings = [], [], []\n",
        "\n",
        "    print(\"Embeddings shape:\", all_embeddings.shape)\n",
        "\n",
        "    for index, filename in enumerate(filenames):\n",
        "        bounding_box = bounding_boxes[filename]\n",
        "\n",
        "        try:\n",
        "            # Load images\n",
        "            img_name = '{}/images/{}.jpg'.format(cub_dataset_dir, filename)\n",
        "            img = get_img(img_name, bounding_box, image_size)\n",
        "\n",
        "            all_embeddings1 = all_embeddings[index, :, :]\n",
        "\n",
        "            embedding_ix = random.randint(0, all_embeddings1.shape[0] - 1)\n",
        "            embedding = all_embeddings1[embedding_ix, :]\n",
        "\n",
        "            X.append(np.array(img))\n",
        "            y.append(class_ids[index])\n",
        "            embeddings.append(embedding)\n",
        "        except Exception as e:\n",
        "            print(e)\n",
        "\n",
        "    X = np.array(X)\n",
        "    y = np.array(y)\n",
        "    embeddings = np.array(embeddings)\n",
        "    return X, y, embeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ODvYdkGtpGKJ",
        "colab_type": "text"
      },
      "source": [
        "# Model Creation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__1ejH3mpFon",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_c(x):\n",
        "    mean = x[:, :128]\n",
        "    log_sigma = x[:, 128:]\n",
        "    stddev = K.exp(log_sigma)\n",
        "    epsilon = K.random_normal(shape=K.constant((mean.shape[1],), dtype='int32'))\n",
        "    c = stddev * epsilon + mean\n",
        "    return c"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYGM28snpPMb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_ca_model():\n",
        "    \"\"\"\n",
        "    Get conditioning augmentation model.\n",
        "    Takes an embedding of shape (1024,) and returns a tensor of shape (256,)\n",
        "    \"\"\"\n",
        "    input_layer = Input(shape=(1024,))\n",
        "    x = Dense(256)(input_layer)\n",
        "    x = LeakyReLU(alpha=0.2)(x)\n",
        "    model = Model(inputs=[input_layer], outputs=[x])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jY_cMcEbpRSW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_embedding_compressor_model():\n",
        "    \"\"\"\n",
        "    Build embedding compressor model\n",
        "    \"\"\"\n",
        "    input_layer = Input(shape=(1024,))\n",
        "    x = Dense(128)(input_layer)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    model = Model(inputs=[input_layer], outputs=[x])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N5BK6rWkpSyQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_stage1_generator():\n",
        "    \"\"\"\n",
        "    Builds a generator model used in Stage-I\n",
        "    \"\"\"\n",
        "    input_layer = Input(shape=(1024,))\n",
        "    x = Dense(256)(input_layer)\n",
        "    mean_logsigma = LeakyReLU(alpha=0.2)(x)\n",
        "\n",
        "    c = Lambda(generate_c)(mean_logsigma)\n",
        "\n",
        "    input_layer2 = Input(shape=(100,))\n",
        "\n",
        "    gen_input = Concatenate(axis=1)([c, input_layer2])\n",
        "\n",
        "    x = Dense(128 * 8 * 4 * 4, use_bias=False)(gen_input)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    x = Reshape((4, 4, 128 * 8), input_shape=(128 * 8 * 4 * 4,))(x)\n",
        "\n",
        "    x = UpSampling2D(size=(2, 2))(x)\n",
        "    x = Conv2D(512, kernel_size=3, padding=\"same\", strides=1, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    x = UpSampling2D(size=(2, 2))(x)\n",
        "    x = Conv2D(256, kernel_size=3, padding=\"same\", strides=1, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    x = UpSampling2D(size=(2, 2))(x)\n",
        "    x = Conv2D(128, kernel_size=3, padding=\"same\", strides=1, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    x = UpSampling2D(size=(2, 2))(x)\n",
        "    x = Conv2D(64, kernel_size=3, padding=\"same\", strides=1, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "\n",
        "    x = Conv2D(3, kernel_size=3, padding=\"same\", strides=1, use_bias=False)(x)\n",
        "    x = Activation(activation='tanh')(x)\n",
        "\n",
        "    stage1_gen = Model(inputs=[input_layer, input_layer2], outputs=[x, mean_logsigma])\n",
        "    return stage1_gen\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xJLclqSTpVGj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_stage1_discriminator():\n",
        "    \"\"\"\n",
        "    Create a model which takes two inputs\n",
        "    1. One from above network\n",
        "    2. One from the embedding layer\n",
        "    3. Concatenate along the axis dimension and feed it to the last module which produces final logits\n",
        "    \"\"\"\n",
        "    input_layer = Input(shape=(64, 64, 3))\n",
        "\n",
        "    x = Conv2D(64, (4, 4),\n",
        "               padding='same', strides=2,\n",
        "               input_shape=(64, 64, 3), use_bias=False)(input_layer)\n",
        "    x = LeakyReLU(alpha=0.2)(x)\n",
        "\n",
        "    x = Conv2D(128, (4, 4), padding='same', strides=2, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = LeakyReLU(alpha=0.2)(x)\n",
        "\n",
        "    x = Conv2D(256, (4, 4), padding='same', strides=2, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = LeakyReLU(alpha=0.2)(x)\n",
        "\n",
        "    x = Conv2D(512, (4, 4), padding='same', strides=2, use_bias=False)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = LeakyReLU(alpha=0.2)(x)\n",
        "\n",
        "    input_layer2 = Input(shape=(4, 4, 128))\n",
        "\n",
        "    merged_input = concatenate([x, input_layer2])\n",
        "\n",
        "    x2 = Conv2D(64 * 8, kernel_size=1,\n",
        "                padding=\"same\", strides=1)(merged_input)\n",
        "    x2 = BatchNormalization()(x2)\n",
        "    x2 = LeakyReLU(alpha=0.2)(x2)\n",
        "    x2 = Flatten()(x2)\n",
        "    x2 = Dense(1)(x2)\n",
        "    x2 = Activation('sigmoid')(x2)\n",
        "\n",
        "    stage1_dis = Model(inputs=[input_layer, input_layer2], outputs=[x2])\n",
        "    return stage1_dis"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NREfScoDpXKb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_adversarial_model(gen_model, dis_model):\n",
        "    input_layer = Input(shape=(1024,))\n",
        "    input_layer2 = Input(shape=(100,))\n",
        "    input_layer3 = Input(shape=(4, 4, 128))\n",
        "\n",
        "    x, mean_logsigma = gen_model([input_layer, input_layer2])\n",
        "\n",
        "    dis_model.trainable = False\n",
        "    valid = dis_model([x, input_layer3])\n",
        "\n",
        "    model = Model(inputs=[input_layer, input_layer2, input_layer3], outputs=[valid, mean_logsigma])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eNBHRmpqpZgo",
        "colab_type": "text"
      },
      "source": [
        "# Defining Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awQ2XK6lpZDr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def KL_loss(y_true, y_pred):\n",
        "    mean = y_pred[:, :128]\n",
        "    logsigma = y_pred[:, :128]\n",
        "    loss = -logsigma + .5 * (-1 + K.exp(2. * logsigma) + K.square(mean))\n",
        "    loss = K.mean(loss)\n",
        "    return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6wkVR9lpd5q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def custom_generator_loss(y_true, y_pred):\n",
        "    # Calculate binary cross entropy loss\n",
        "    return K.binary_crossentropy(y_true, y_pred)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1wGTNAxZpfoc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def save_rgb_img(img, path):\n",
        "    \"\"\"\n",
        "    Save an rgb image\n",
        "    \"\"\"\n",
        "    fig = plt.figure()\n",
        "    ax = fig.add_subplot(1, 1, 1)\n",
        "    ax.imshow(img)\n",
        "    ax.axis(\"off\")\n",
        "    ax.set_title(\"Image\")\n",
        "\n",
        "    plt.savefig(path)\n",
        "    plt.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfdGkJj0phPb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def write_log(callback, name, loss, batch_no):\n",
        "    \"\"\"\n",
        "    Write training summary to TensorBoard\n",
        "    \"\"\"\n",
        "    summary = tf.Summary()\n",
        "    summary_value = summary.value.add()\n",
        "    summary_value.simple_value = loss\n",
        "    summary_value.tag = name\n",
        "    callback.writer.add_summary(summary, batch_no)\n",
        "    callback.writer.flush()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qTgvkqyvptfn",
        "colab_type": "text"
      },
      "source": [
        "# Downloading of dataset in colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFIC465b2w0u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdxKQ0663T_C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHohaW6L2wyW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tarfile\n",
        "tar = tarfile.open(\"CUB_200_2011.tgz\")\n",
        "tar.extractall()\n",
        "tar.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jZ6Xgi--2wvp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0fTLsFUb4ftd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cd CUB_200_2011"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_xbNKn-4f5r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AdEAGWQE4gB-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.chdir(\"/content/\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BMrVDAC4gMN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RLWyhSzSZ81s",
        "colab_type": "text"
      },
      "source": [
        "# **My Edit__dir_create**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6nGGGbSaCbu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "result_path = '/content/results'+currentmodel+'/'\n",
        "!mkdir -p /content/logs/\n",
        "!mkdir -p $result_path"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f6aAxfRYyPec",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir -p $dirpath$createdir"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JWRKO6IfpkFt",
        "colab_type": "text"
      },
      "source": [
        "# Main File"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Pjt8uWEpjsS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if __name__ == '__main__':\n",
        "    gen_loss = open('/content/gen_loss_'+currentmodel+'.txt','a')\n",
        "    dis_loss = open('/content/dis_loss_'+currentmodel+'.txt','a')\n",
        "    data_dir = \"/content/birds/\"\n",
        "    train_dir = data_dir + \"/train\"\n",
        "    test_dir = data_dir + \"/test\"\n",
        "    image_size = 64\n",
        "    batch_size = 64\n",
        "    z_dim = 100\n",
        "    stage1_generator_lr = 0.0002\n",
        "    stage1_discriminator_lr = 0.0002\n",
        "    stage1_lr_decay_step = 600\n",
        "    epochs = SET_EPOCHS\n",
        "    condition_dim = 128\n",
        "\n",
        "    embeddings_file_path_train = train_dir + \"/char-CNN-RNN-embeddings.pickle\"\n",
        "    embeddings_file_path_test = test_dir + \"/char-CNN-RNN-embeddings.pickle\"\n",
        "\n",
        "    filenames_file_path_train = train_dir + \"/filenames.pickle\"\n",
        "    filenames_file_path_test = test_dir + \"/filenames.pickle\"\n",
        "\n",
        "    class_info_file_path_train = train_dir + \"/class_info.pickle\"\n",
        "    class_info_file_path_test = test_dir + \"/class_info.pickle\"\n",
        "\n",
        "    cub_dataset_dir = \"/content/CUB_200_2011\"\n",
        "    \n",
        "    # Define optimizers\n",
        "    dis_optimizer = Adam(lr=stage1_discriminator_lr, beta_1=0.5, beta_2=0.999)\n",
        "    gen_optimizer = Adam(lr=stage1_generator_lr, beta_1=0.5, beta_2=0.999)\n",
        "\n",
        "    \"\"\"\"\n",
        "    Load datasets\n",
        "    \"\"\"\n",
        "    X_train, y_train, embeddings_train = load_dataset(filenames_file_path=filenames_file_path_train,\n",
        "                                                      class_info_file_path=class_info_file_path_train,\n",
        "                                                      cub_dataset_dir=cub_dataset_dir,\n",
        "                                                      embeddings_file_path=embeddings_file_path_train,\n",
        "                                                      image_size=(64, 64))\n",
        "\n",
        "    X_test, y_test, embeddings_test = load_dataset(filenames_file_path=filenames_file_path_test,\n",
        "                                                   class_info_file_path=class_info_file_path_test,\n",
        "                                                   cub_dataset_dir=cub_dataset_dir,\n",
        "                                                   embeddings_file_path=embeddings_file_path_test,\n",
        "                                                   image_size=(64, 64))\n",
        "\n",
        "    \"\"\"\n",
        "    Build and compile networks\n",
        "    \"\"\"\n",
        "    print(\"===========build models==================\")\n",
        "    try:\n",
        "      ca_model = load_model('/content/models/ca_model_'+previousmodel+'.h5')\n",
        "      print('load ca_model')\n",
        "    except:\n",
        "      ca_model = build_ca_model()\n",
        "      ca_model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\")\n",
        "      print('create ca_model')\n",
        "\n",
        "    try:\n",
        "      stage1_dis = load_model('/content/models/stage1_dis_model_'+previousmodel+'.h5')\n",
        "      print('load stage1_dis')\n",
        "    except:\n",
        "      stage1_dis = build_stage1_discriminator()\n",
        "      stage1_dis.compile(loss='binary_crossentropy', optimizer=dis_optimizer)\n",
        "      print('create stage1_dis')\n",
        "\n",
        "    try:\n",
        "      stage1_gen = load_model('/content/models/stage1_gen_model_'+previousmodel+'.h5')\n",
        "      print('load stage1_gen')\n",
        "    except:\n",
        "      stage1_gen = build_stage1_generator()\n",
        "      stage1_gen.compile(loss=\"mse\", optimizer=gen_optimizer)\n",
        "      print('create stage1_gen')\n",
        "\n",
        "    try:\n",
        "      embedding_compressor_model = load_model('/content/models/embedding_compressor_model_'+previousmodel+'.h5')\n",
        "      print('load embedding_compressor_model')\n",
        "    except:\n",
        "      embedding_compressor_model = build_embedding_compressor_model()\n",
        "      embedding_compressor_model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\")\n",
        "      print('create embedding_compressor_model')\n",
        "\n",
        "    try:\n",
        "      # load json and create model\n",
        "      json_file = open('/content/models/adversarial_model_json_'+previousmodel+'.json', 'r')\n",
        "      loaded_model_json = json_file.read()\n",
        "      json_file.close()\n",
        "      adversarial_model = model_from_json(loaded_model_json)\n",
        "      # load weights into new model\n",
        "      adversarial_model.load_weights('/content/models/adversarial_model_weight_'+previousmodel+'.h5')\n",
        "      adversarial_model.compile(loss=['binary_crossentropy', KL_loss], loss_weights=[1, 2.0],optimizer=gen_optimizer, metrics=None)\n",
        "      print('load adversarial_model')\n",
        "    except:\n",
        "      adversarial_model = build_adversarial_model(gen_model=stage1_gen, dis_model=stage1_dis)\n",
        "      adversarial_model.compile(loss=['binary_crossentropy', KL_loss], loss_weights=[1, 2.0],optimizer=gen_optimizer, metrics=None)\n",
        "      print('create adversarial_model')\n",
        "\n",
        "    tensorboard = TensorBoard(log_dir=\"logs/\".format(time.time()))\n",
        "    tensorboard.set_model(stage1_gen)\n",
        "    tensorboard.set_model(stage1_dis)\n",
        "    tensorboard.set_model(ca_model)\n",
        "    tensorboard.set_model(embedding_compressor_model)\n",
        "\n",
        "    \n",
        "\n",
        "\n",
        "    # Generate an array containing real and fake values\n",
        "    # Apply label smoothing as well\n",
        "    real_labels = np.ones((batch_size, 1), dtype=float) * 0.9\n",
        "    fake_labels = np.zeros((batch_size, 1), dtype=float) * 0.1\n",
        "\n",
        "    \n",
        "    for epoch in range(epochs):\n",
        "        print(\"========================================\")\n",
        "        print(\"Epoch is:\", epoch)\n",
        "        print(\"Number of batches\", int(X_train.shape[0] / batch_size))\n",
        "\n",
        "        gen_losses = []\n",
        "        dis_losses = []\n",
        "\n",
        "        # Load data and train model\n",
        "        number_of_batches = int(X_train.shape[0] / batch_size)\n",
        "        for index in range(number_of_batches):\n",
        "            print(\"Batch:{}\".format(index+1))\n",
        "            \n",
        "            \"\"\"\n",
        "            Train the discriminator network\n",
        "            \"\"\"\n",
        "            # Sample a batch of data\n",
        "            z_noise = np.random.normal(0, 1, size=(batch_size, z_dim))\n",
        "            image_batch = X_train[index * batch_size:(index + 1) * batch_size]\n",
        "            embedding_batch = embeddings_train[index * batch_size:(index + 1) * batch_size]\n",
        "            image_batch = (image_batch - 127.5) / 127.5\n",
        "\n",
        "            # Generate fake images\n",
        "            fake_images, _ = stage1_gen.predict([embedding_batch, z_noise], verbose=3)\n",
        "\n",
        "            # Generate compressed embeddings\n",
        "            compressed_embedding = embedding_compressor_model.predict_on_batch(embedding_batch)\n",
        "            compressed_embedding = np.reshape(compressed_embedding, (-1, 1, 1, condition_dim))\n",
        "            compressed_embedding = np.tile(compressed_embedding, (1, 4, 4, 1))\n",
        "\n",
        "            dis_loss_real = stage1_dis.train_on_batch([image_batch, compressed_embedding],\n",
        "                                                      np.reshape(real_labels, (batch_size, 1)))\n",
        "            dis_loss_fake = stage1_dis.train_on_batch([fake_images, compressed_embedding],\n",
        "                                                      np.reshape(fake_labels, (batch_size, 1)))\n",
        "            dis_loss_wrong = stage1_dis.train_on_batch([image_batch[:(batch_size - 1)], compressed_embedding[1:]],\n",
        "                                                       np.reshape(fake_labels[1:], (batch_size-1, 1)))\n",
        "\n",
        "            d_loss = 0.5 * np.add(dis_loss_real, 0.5 * np.add(dis_loss_wrong, dis_loss_fake))\n",
        "\n",
        "            #print(\"d_loss_real:{}\".format(dis_loss_real))\n",
        "            #print(\"d_loss_fake:{}\".format(dis_loss_fake))\n",
        "            #print(\"d_loss_wrong:{}\".format(dis_loss_wrong))\n",
        "            #print(\"d_loss:{}\".format(d_loss))\n",
        "\n",
        "            \"\"\"\n",
        "            Train the generator network \n",
        "            \"\"\"\n",
        "            g_loss = adversarial_model.train_on_batch([embedding_batch, z_noise, compressed_embedding],[K.ones((batch_size, 1)) * 0.9, K.ones((batch_size, 256)) * 0.9])\n",
        "            print(\"g_loss:{}\".format(g_loss))\n",
        "\n",
        "            dis_losses.append(d_loss)\n",
        "            gen_losses.append(g_loss)\n",
        "\n",
        "        \"\"\"\n",
        "        Save losses to Tensorboard after each epoch\n",
        "        \"\"\"\n",
        "        #write_log(tensorboard, 'discriminator_loss', np.mean(dis_losses), epoch)\n",
        "        #write_log(tensorboard, 'generator_loss', np.mean(gen_losses[0]), epoch)\n",
        "        \n",
        "        dis_loss.write(str(np.mean(dis_losses))+'\\n')\n",
        "        gen_loss.write(str(np.mean(gen_losses[0]))+'\\n')\n",
        "\n",
        "        # Generate and save images after every 2nd epoch\n",
        "        if epoch % 2 == 0:\n",
        "            # z_noise2 = np.random.uniform(-1, 1, size=(batch_size, z_dim))\n",
        "            z_noise2 = np.random.normal(0, 1, size=(batch_size, z_dim))\n",
        "            embedding_batch = embeddings_test[0:batch_size]\n",
        "            fake_images, _ = stage1_gen.predict_on_batch([embedding_batch, z_noise2])\n",
        "\n",
        "            # Save images\n",
        "            for i, img in enumerate(fake_images[:10]):\n",
        "                save_rgb_img(img, \"results\"+currentmodel+\"/gen_{}_{}.png\".format(epoch, i))\n",
        "    \n",
        "    ### close file\n",
        "    dis_loss.close()\n",
        "    gen_loss.close()\n",
        "    \n",
        "    '''\n",
        "    # Save weights\n",
        "    stage1_gen.save_weights(\"stage1_gen_1905.h5\")\n",
        "    stage1_dis.save_weights(\"stage1_dis_1905.h5\")\n",
        "    #!cp -r /content/stage1_gen.h5 /content/drive/'My Drive'/GAN/date1105/\n",
        "    #!cp -r /content/stage1_dis.h5 /content/drive/'My Drive'/GAN/date1105/\n",
        "    #!cp -r /content/logs /content/drive/'My Drive'/GAN/date1105/\n",
        "    #!cp -r /content/results /content/drive/'My Drive'/GAN/date1105/\n",
        "\n",
        "    # Save MOdels\n",
        "    stage1_gen.save(\"stage1_gen_model_1905.h5\")\n",
        "    stage1_dis.save(\"stage1_dis_model_1905.h5\")\n",
        "    ca_model.save(\"ca_model_1905.h5\")\n",
        "    embedding_compressor_model.save(\"embedding_compressor_model_1905.h5\")    \n",
        "\n",
        "    adversarial_model.save_weights(\"adversarial_model_weight_1905.h5\")\n",
        "    adversarial_model_json = adversarial_model.to_json()\n",
        "    with open(\"adversarial_model_json_1905.json\", \"w\") as json_file:\n",
        "      json_file.write(adversarial_model_json)\n",
        "    '''\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qsOdcdLlg9Zw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make directory to save current model and weights\n",
        "!mkdir -p /content/currentweights/\n",
        "!mkdir -p /content/currentmodels/\n",
        "\n",
        "# Save weights\n",
        "stage1_gen.save_weights(\"currentweights/stage1_gen_\"+currentmodel+\".h5\")\n",
        "stage1_dis.save_weights(\"currentweights/stage1_dis_\"+currentmodel+\".h5\")\n",
        "\n",
        "\n",
        "# Save MOdels\n",
        "stage1_gen.save(\"currentmodels/stage1_gen_model_\"+currentmodel+\".h5\")\n",
        "stage1_dis.save(\"currentmodels/stage1_dis_model_\"+currentmodel+\".h5\")\n",
        "ca_model.save(\"currentmodels/ca_model_\"+currentmodel+\".h5\")\n",
        "embedding_compressor_model.save(\"currentmodels/embedding_compressor_model_\"+currentmodel+\".h5\")    \n",
        "\n",
        "adversarial_model.save_weights(\"currentmodels/adversarial_model_weight_\"+currentmodel+\".h5\")\n",
        "adversarial_model_json = adversarial_model.to_json()\n",
        "with open(\"currentmodels/adversarial_model_json_\"+currentmodel+\".json\", \"w\") as json_file:\n",
        "  json_file.write(adversarial_model_json)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wF7oHRPebAP8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gen_los = '/content/gen_loss_'+currentmodel+'.txt'\n",
        "dis_los = '/content/dis_loss_'+currentmodel+'.txt'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XNOjidCkek1X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######### copy weights\n",
        "!cp -r /content/currentweights/* $dirpath$weightpath\n",
        "\n",
        "######## copy loss\n",
        "!cp -r $gen_los $dirpath$weightpath\n",
        "!cp -r $dis_los $dirpath$weightpath\n",
        "\n",
        "######## copy results\n",
        "!cp -r /content/logs /content/drive/'My Drive'/GAN/$weightpath\n",
        "!cp -r $result_path /content/drive/'My Drive'/GAN/$weightpath\n",
        "\n",
        "###### copy model \n",
        "!cp -r /content/currentmodels/* $dirpath$createdir"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}