5_neural_nets_with_keras.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "5_neural_nets_with_keras.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "nav_menu": {
      "height": "264px",
      "width": "369px"
    },
    "toc": {
      "navigate_menu": true,
      "number_sections": true,
      "sideBar": true,
      "threshold": 6,
      "toc_cell": false,
      "toc_section_display": "block",
      "toc_window_display": false
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "piBSRAqc96nN"
      },
      "source": [
        "**Chapter 10 – Introduction to Artificial Neural Networks with Keras**\n",
        "\n",
        "_This notebook contains all the sample code and solutions to the exercises in chapter 10._"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3TTeAuN96nU"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6fZRoRpY96nV"
      },
      "source": [
        "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3_Ulmke96nV"
      },
      "source": [
        "# Python ≥3.5 is required\n",
        "import sys\n",
        "assert sys.version_info >= (3, 5)\n",
        "\n",
        "# Scikit-Learn ≥0.20 is required\n",
        "import sklearn\n",
        "assert sklearn.__version__ >= \"0.20\"\n",
        "\n",
        "try:\n",
        "    # %tensorflow_version only exists in Colab.\n",
        "    %tensorflow_version 2.x\n",
        "except Exception:\n",
        "    pass\n",
        "\n",
        "# TensorFlow ≥2.0 is required\n",
        "import tensorflow as tf\n",
        "assert tf.__version__ >= \"2.0\"\n",
        "\n",
        "# Common imports\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "# to make this notebook's output stable across runs\n",
        "np.random.seed(42)\n",
        "\n",
        "# To plot pretty figures\n",
        "%matplotlib inline\n",
        "import matplotlib as mpl\n",
        "import matplotlib.pyplot as plt\n",
        "mpl.rc('axes', labelsize=14)\n",
        "mpl.rc('xtick', labelsize=12)\n",
        "mpl.rc('ytick', labelsize=12)\n",
        "\n",
        "# Where to save the figures\n",
        "PROJECT_ROOT_DIR = \".\"\n",
        "CHAPTER_ID = \"ann\"\n",
        "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
        "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
        "\n",
        "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
        "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
        "    print(\"Saving figure\", fig_id)\n",
        "    if tight_layout:\n",
        "        plt.tight_layout()\n",
        "    plt.savefig(path, format=fig_extension, dpi=resolution)\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AKK8dS-C96nW"
      },
      "source": [
        "# Perceptrons"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T2GbE_Wj96nW"
      },
      "source": [
        "**Note**: we set `max_iter` and `tol` explicitly to avoid warnings about the fact that their default value will change in future versions of Scikit-Learn."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tut21LOA-I2r"
      },
      "source": [
        "**Task 1:** Fit the iris dataset into a Perceptron layer and predict the class of a sample with length 2 and width 0.5. Use: \r\n",
        "'max_iter=1000', 'tol =1e-3' and 'random_state= 42'.\r\n",
        "\r\n",
        "\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bc419YFJ96nX"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.datasets import load_iris\n",
        "from sklearn.linear_model import Perceptron\n",
        "\n",
        "iris = load_iris()\n",
        "X = iris.data[:, (2, 3)]  # petal length, petal width\n",
        "y = (iris.target == 0).astype(np.int)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-5wg3vIj3gxL"
      },
      "source": [
        "#per_clf = \r\n",
        "\r\n",
        "#y_pred = "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RU8nJGQo96nY"
      },
      "source": [
        "#y_pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzqgH5uG96nZ"
      },
      "source": [
        "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
        "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
        "\n",
        "axes = [0, 5, 0, 2]\n",
        "\n",
        "x0, x1 = np.meshgrid(\n",
        "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
        "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
        "    )\n",
        "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
        "y_predict = per_clf.predict(X_new)\n",
        "zz = y_predict.reshape(x0.shape)\n",
        "\n",
        "plt.figure(figsize=(10, 4))\n",
        "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
        "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
        "\n",
        "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
        "from matplotlib.colors import ListedColormap\n",
        "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
        "\n",
        "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
        "plt.xlabel(\"Petal length\", fontsize=14)\n",
        "plt.ylabel(\"Petal width\", fontsize=14)\n",
        "plt.legend(loc=\"lower right\", fontsize=14)\n",
        "plt.axis(axes)\n",
        "\n",
        "save_fig(\"perceptron_iris_plot\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c7AzNDaU-xOV"
      },
      "source": [
        "**Task 2:** Elaborate **in the report** on the difference between a perceptron and logistic regression."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AZr175SP96na"
      },
      "source": [
        "# Activation functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBhSPTlaDZcE"
      },
      "source": [
        "**Task 3:** Briefly describe **in the report** the role of activation functions within a neural network. What would be a neural network without these functions? "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4zZHVAV096nd"
      },
      "source": [
        "# Building an Image Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6d_vhWv96nd"
      },
      "source": [
        "First let's import TensorFlow and Keras."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2XiMCMx96nd"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "kSUOusSb96nd",
        "outputId": "3d533c5f-89b8-4874-d841-513c98d28e82"
      },
      "source": [
        "tf.__version__"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.4.1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "qClTS5Qo96nd",
        "outputId": "bb8da60d-e856-4835-9823-8589ef821352"
      },
      "source": [
        "keras.__version__"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.4.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0IC001Fj96ne"
      },
      "source": [
        "Let's start by loading the fashion MNIST dataset. Keras has a number of functions to load popular datasets in `keras.datasets`. The dataset is already split for you between a training set and a test set, but it can be useful to split the training set further to have a validation set:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1Vd2nar96ne"
      },
      "source": [
        "fashion_mnist = keras.datasets.fashion_mnist\n",
        "(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bm2mJI2v96ne"
      },
      "source": [
        "The training set contains 60,000 grayscale images, each 28x28 pixels:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mKeNURr096nf"
      },
      "source": [
        "X_train_full.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2q7eH1GS96nf"
      },
      "source": [
        "Each pixel intensity is represented as a byte (0 to 255):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JpJBj89g96ng"
      },
      "source": [
        "X_train_full.dtype"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "up_jreEp96ng"
      },
      "source": [
        "Let's split the full training set into a validation set and a (smaller) training set. We also scale the pixel intensities down to the 0-1 range and convert them to floats, by dividing by 255."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAZzoYRD96ng"
      },
      "source": [
        "X_valid, X_train = X_train_full[:5000] / 255., X_train_full[5000:] / 255.\n",
        "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
        "X_test = X_test / 255."
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IG-ZXYbM96ng"
      },
      "source": [
        "You can plot an image using Matplotlib's `imshow()` function, with a `'binary'`\n",
        " color map:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5qfRYzl96ng"
      },
      "source": [
        "plt.imshow(X_train[0], cmap=\"binary\")\n",
        "plt.axis('off')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0HIhKfBK96nh"
      },
      "source": [
        "The labels are the class IDs (represented as uint8), from 0 to 9:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CC0hpimX96nh"
      },
      "source": [
        "y_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XuQH9NeY96nh"
      },
      "source": [
        "Here are the corresponding class names:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6DnDTta596nh"
      },
      "source": [
        "class_names = [\"T-shirt/top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\",\n",
        "               \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle boot\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Z3ctcPE96ni"
      },
      "source": [
        "So the first image in the training set is a coat:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wo3nXTyr96ni"
      },
      "source": [
        "class_names[y_train[0]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CSVf64Kv96ni"
      },
      "source": [
        "The validation set contains 5,000 images, and the test set contains 10,000 images:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3DPLv7496ni"
      },
      "source": [
        "X_valid.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBVt2-n196nj"
      },
      "source": [
        "X_test.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cr7f-I2i96nj"
      },
      "source": [
        "Let's take a look at a sample of the images in the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OiwEaciG96nj"
      },
      "source": [
        "n_rows = 4\n",
        "n_cols = 10\n",
        "plt.figure(figsize=(n_cols * 1.2, n_rows * 1.2))\n",
        "for row in range(n_rows):\n",
        "    for col in range(n_cols):\n",
        "        index = n_cols * row + col\n",
        "        plt.subplot(n_rows, n_cols, index + 1)\n",
        "        plt.imshow(X_train[index], cmap=\"binary\", interpolation=\"nearest\")\n",
        "        plt.axis('off')\n",
        "        plt.title(class_names[y_train[index]], fontsize=12)\n",
        "plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
        "save_fig('fashion_mnist_plot', tight_layout=False)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t1G5vYM4AEYy"
      },
      "source": [
        "**Training a neural network using Keras:** In the cells below we  build an deep neural network model using Keras Sequential tool. The input is an image of shape 28 by 28. We use a network with 2 hidden layers with 300 and 100 neurons, respectively. The 'softmax' output activation function is used for multi-label classification.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFYpKULg96nk"
      },
      "source": [
        "keras.backend.clear_session()\n",
        "np.random.seed(42)\n",
        "tf.random.set_seed(42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHTrilwL96nk"
      },
      "source": [
        "model = keras.models.Sequential([\r\n",
        "    keras.layers.Flatten(input_shape=[28, 28]),\r\n",
        "    keras.layers.Dense(300, activation=\"relu\"),\r\n",
        "    keras.layers.Dense(100, activation=\"relu\"),\r\n",
        "    keras.layers.Dense(10, activation=\"softmax\")\r\n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6wLCkUc96nk"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rt04f4Ug96nl"
      },
      "source": [
        "keras.utils.plot_model(model, \"my_fashion_mnist_model.png\", show_shapes=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DqQcMXAC96nl"
      },
      "source": [
        "hidden1 = model.layers[1]\n",
        "hidden1.name"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVQqeMeS96nl"
      },
      "source": [
        "model.get_layer(hidden1.name) is hidden1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eR5m6LjX96nl"
      },
      "source": [
        "weights, biases = hidden1.get_weights()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ETO_Mi2f96nm"
      },
      "source": [
        "weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i-AWFPWX96nm",
        "outputId": "4e50d7fd-0c8a-4bbf-c646-9e9c17af21ae"
      },
      "source": [
        "weights.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(784, 300)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qb7RJWWg96nm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "outputId": "09812d15-53ab-42e4-f1e2-b948cab5117d"
      },
      "source": [
        "biases"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-1627f7cda061>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbiases\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'biases' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SRfITXCA96nm"
      },
      "source": [
        "biases.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pDu2dLD8BbeJ"
      },
      "source": [
        "**Compilation:** We compile the model using the \"sparse_categorical_crossentropy\" loss function and \"accuracy\" as metric for a multi-label classification task. In addition, we choose the \"sgd\" optimizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iha1BNF296nn"
      },
      "source": [
        "model.compile(loss=\"sparse_categorical_crossentropy\",\r\n",
        "              optimizer=\"sgd\",\r\n",
        "              metrics=[\"accuracy\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CrUl_WUBw0E"
      },
      "source": [
        "**Note:** Once compiled, we use model.fit to train the model for at least 30 epochs. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z01D2HtY6B4r"
      },
      "source": [
        "We save the most crucial parameters (['loss', 'accuracy', 'val_loss', 'val_accuracy']) in a dictionary named \"history\". "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "niNnbD7U96no"
      },
      "source": [
        "history = model.fit(X_train, y_train, epochs=30,\r\n",
        "                    validation_data=(X_valid, y_valid))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ZsFB62z96no"
      },
      "source": [
        "history.params"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdaUfmYb96np"
      },
      "source": [
        "print(history.epoch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O3tukEej96np"
      },
      "source": [
        "history.history.keys()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CCv_GPMi96np"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "pd.DataFrame(history.history).plot(figsize=(8, 5))\n",
        "plt.grid(True)\n",
        "plt.gca().set_ylim(0, 1)\n",
        "save_fig(\"keras_learning_curves_plot\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p2WdGqAlC0bm"
      },
      "source": [
        "**Task 4:** Validate your model using model.evaluate on the test set made of X_test and y_test. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfC_sLA_96nq"
      },
      "source": [
        "# evaluate model on  test set"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zM8ssoN0FjYS"
      },
      "source": [
        "**Task 5:** Select the first three samples from the test set and predict their corresponding classes using model.predict_classes . Then print the names/categories of the elements in  question (Eg. \"Pants\", \"trouser\") "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7C0en5Jk96nq"
      },
      "source": [
        "X_new = X_test[:3]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ySYN2iHW96nq"
      },
      "source": [
        "**Warning**: `model.predict_classes(X_new)` is deprecated. It is replaced with `np.argmax(model.predict(X_new), axis=-1)`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vzDuZw896nq"
      },
      "source": [
        "#y_pred = \n",
        "#y_pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eij41cJh96nr"
      },
      "source": [
        "y_new = y_test[:3]\n",
        "y_new"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MloueY3o96nr"
      },
      "source": [
        "plt.figure(figsize=(7.2, 2.4))\n",
        "for index, image in enumerate(X_new):\n",
        "    plt.subplot(1, 3, index + 1)\n",
        "    plt.imshow(image, cmap=\"binary\", interpolation=\"nearest\")\n",
        "    plt.axis('off')\n",
        "    plt.title(class_names[y_test[index]], fontsize=12)\n",
        "plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
        "save_fig('fashion_mnist_images_plot', tight_layout=False)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wdQPEnaa96nr"
      },
      "source": [
        "# Regression MLP"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qMxzRNLS96ns"
      },
      "source": [
        "Let's load, split and scale the California housing dataset (the original one, not the modified one as in chapter 2):"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wJiLPiB5Gzht"
      },
      "source": [
        "**Task 6:** Scale your training, validation and test feature matrices using scikitlearn's StandardScaler. (Dcoumentation and example on this [link](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html))"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZJ3bFpfU96ns"
      },
      "source": [
        "from sklearn.datasets import fetch_california_housing\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "housing = fetch_california_housing()\n",
        "\n",
        "X_train_full, X_test, y_train_full, y_test = train_test_split(housing.data, housing.target, random_state=42)\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full, random_state=42)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O0ehUJYh7Dcw"
      },
      "source": [
        "#scaler = \r\n",
        "#X_train = \r\n",
        "#X_valid = \r\n",
        "#X_test = "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tv60cUzR96ns"
      },
      "source": [
        "np.random.seed(42)\n",
        "tf.random.set_seed(42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1NgYV0XyIlfr"
      },
      "source": [
        "**Task 7:** Build a Neural Network with one hidden layer with 30 neurons. The output layer has one neuron, which is the regression value. Compile, fit then train this network while choosing the suitable loss function and the SGD optimizer. \r\n",
        "\r\n",
        "**N.B:**  This task is similar to task 5 except that we now do regression not classification. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wj5v_bRU96ns"
      },
      "source": [
        "#model = #\n",
        "\n",
        "\n",
        "#...\n",
        "#history = \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F9B_e4Tg96ns"
      },
      "source": [
        "plt.plot(pd.DataFrame(history.history))\n",
        "plt.grid(True)\n",
        "plt.gca().set_ylim(0, 1)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bSADjQQDJLgh"
      },
      "source": [
        "**Task 8:** Validate your model using model.evaluate on a test set. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3IL0WlYJCx6"
      },
      "source": [
        "#mse_test = \r\n",
        "#X_new = \r\n",
        "#y_pred = "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gk42kovV96nt"
      },
      "source": [
        "y_pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b_YAL3g-96nx"
      },
      "source": [
        "# Saving the model weights for future use"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CIMTsRwT96ny"
      },
      "source": [
        "model.save(\"my_keras_model.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrgCYxY596ny"
      },
      "source": [
        "model = keras.models.load_model(\"my_keras_model.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XeUU8hqO96ny"
      },
      "source": [
        "model.predict(X_new)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JTn29YSI96ny"
      },
      "source": [
        "model.save_weights(\"my_keras_weights.ckpt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Ebtp4OJ96nz"
      },
      "source": [
        "model.load_weights(\"my_keras_weights.ckpt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6EFNcwqg96n9"
      },
      "source": [
        "# Optional task"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8cO1eZvL96n-"
      },
      "source": [
        "*Exercise: Train a deep MLP on the MNIST dataset (you can load it using `keras.datasets.mnist.load_data()`. See if you can get over 98% precision. Try searching for the optimal learning rate by using the approach presented in this chapter (i.e., by growing the learning rate exponentially, plotting the loss, and finding the point where the loss shoots up). Try adding all the bells and whistles—save checkpoints, use early stopping, and plot learning curves using TensorBoard.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gRSB5E6q96n-"
      },
      "source": [
        "Let's load the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZxFNmxd96n-"
      },
      "source": [
        "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uFbDQB_e96n-"
      },
      "source": [
        "Just like for the Fashion MNIST dataset, the MNIST training set contains 60,000 grayscale images, each 28x28 pixels:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKu9ctYq96n-"
      },
      "source": [
        "X_train_full.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CjAPFHCy96n_"
      },
      "source": [
        "Each pixel intensity is also represented as a byte (0 to 255):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FKsd4QsX96n_"
      },
      "source": [
        "X_train_full.dtype"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pkMt_Ore96n_"
      },
      "source": [
        "Let's split the full training set into a validation set and a (smaller) training set. We also scale the pixel intensities down to the 0-1 range and convert them to floats, by dividing by 255, just like we did for Fashion MNIST:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGsM7Im596n_"
      },
      "source": [
        "X_valid, X_train = X_train_full[:5000] / 255., X_train_full[5000:] / 255.\n",
        "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
        "X_test = X_test / 255."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBVreanI96oA"
      },
      "source": [
        "Let's plot an image using Matplotlib's `imshow()` function, with a `'binary'`\n",
        " color map:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2_wzz3Gp96oA"
      },
      "source": [
        "plt.imshow(X_train[0], cmap=\"binary\")\n",
        "plt.axis('off')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rYEP13UT96oA"
      },
      "source": [
        "The labels are the class IDs (represented as uint8), from 0 to 9. Conveniently, the class IDs correspond to the digits represented in the images, so we don't need a `class_names` array:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wG8kJsZ796oA"
      },
      "source": [
        "y_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2a7wQTky96oB"
      },
      "source": [
        "The validation set contains 5,000 images, and the test set contains 10,000 images:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FA48HsSM96oB"
      },
      "source": [
        "X_valid.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ibyP5j4496oB"
      },
      "source": [
        "X_test.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BB27dAFd96oB"
      },
      "source": [
        "Let's take a look at a sample of the images in the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "388NsEEo96oB"
      },
      "source": [
        "n_rows = 4\n",
        "n_cols = 10\n",
        "plt.figure(figsize=(n_cols * 1.2, n_rows * 1.2))\n",
        "for row in range(n_rows):\n",
        "    for col in range(n_cols):\n",
        "        index = n_cols * row + col\n",
        "        plt.subplot(n_rows, n_cols, index + 1)\n",
        "        plt.imshow(X_train[index], cmap=\"binary\", interpolation=\"nearest\")\n",
        "        plt.axis('off')\n",
        "        plt.title(y_train[index], fontsize=12)\n",
        "plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "duYAqe7U96oC"
      },
      "source": [
        "Let's build a simple dense network and find the optimal learning rate. We will need a callback to grow the learning rate at each iteration. It will also record the learning rate and the loss at each iteration:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dXlp1yv2lGJ7"
      },
      "source": [
        "**Task 9:** Using the same methodology as earlier tasks, clear the session, set random seed to 42 and build a neural network with 2 hidden layers to classify MNIST images. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g8Q2kkAs96oC"
      },
      "source": [
        "#"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nr3tFAgD96oC"
      },
      "source": [
        "#model ="
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8_Ex9OIR96oD"
      },
      "source": [
        "# compile model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uDHoyVX696oD"
      },
      "source": [
        "Now let's train the model for just 1 epoch:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkLRa1ik96oD"
      },
      "source": [
        "#history ="
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QFQ7H2SP96oF"
      },
      "source": [
        "run_index = 1 # increment this at every run\n",
        "run_logdir = os.path.join(os.curdir, \"my_mnist_logs\", \"run_{:03d}\".format(run_index))\n",
        "run_logdir"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXTWKMua96oF"
      },
      "source": [
        "early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\n",
        "checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_mnist_model.h5\", save_best_only=True)\n",
        "tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\n",
        "\n",
        "history = model.fit(X_train, y_train, epochs=100,\n",
        "                    validation_data=(X_valid, y_valid),\n",
        "                    callbacks=[checkpoint_cb, early_stopping_cb, tensorboard_cb])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fOSz5oRF96oF"
      },
      "source": [
        "model = keras.models.load_model(\"my_mnist_model.h5\") # rollback to best model\n",
        "model.evaluate(X_test, y_test)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}