workshop_3_dnn_implementation_complete.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create anaconda environment\n",
    "<br>\n",
    "```bash\n",
    "conda create -n ml python=3.7.5 jupyter\n",
    "```\n",
    "Install fastai library\n",
    "<br>\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y_str = fetch_openml('mnist_784', version=1, return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img1 = X[1].reshape(28, 28)\n",
    "plt.imshow(img1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_int = y_str.astype(int)\n",
    "\n",
    "def encode_idx(idx):\n",
    "    cd = np.zeros(10)\n",
    "    cd[idx] = 1\n",
    "    \n",
    "    return cd\n",
    "    \n",
    "y_v = np.array([encode_idx(i) for i in y_int])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_norm = X / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = X_norm[:60000], X[60000:]\n",
    "y_train, y_test = y_v[:60000], y_v[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "img_norm = X_train[0].reshape(28, 28)\n",
    "plt.imshow(img_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_train, y_train)]\n",
    "test_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_test, y_test)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z:np.ndarray):\n",
    "    return 1.0 / (1.0 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define derivative of sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_der(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement the linear layer of our neural network\n",
    "<br>\n",
    "$$\n",
    "z = Wx + b\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(object):\n",
    "    \n",
    "    def __init__(self, din:int, dout:int, bias:bool=True):\n",
    "        super().__init__()\n",
    "        self.din = din\n",
    "        self.dout = dout\n",
    "        self.bias = bias\n",
    "        self.W = np.random.randn(dout, din)\n",
    "        self.b = np.random.randn(dout, 1)\n",
    "        self.a = None\n",
    "        self.z = None\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "        \n",
    "    def forward(self, a:np.ndarray):\n",
    "        self.a = a\n",
    "        self.z = self.W @ a + self.b\n",
    "        \n",
    "        return self.z\n",
    "    \n",
    "    def backward(self, delta_n:np.ndarray, z_p:np.ndarray):\n",
    "        sd = sigmoid_der(z_p)\n",
    "        delta = (self.W.T @ delta_n) * sd\n",
    "        self.der_W += delta_n @ self.a.T\n",
    "        self.der_b += delta_n\n",
    "        \n",
    "        return delta\n",
    "    \n",
    "    def zero_grad(self):\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return f'Linear({self.din}, {self.dout})'\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return self.__str__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here din is dimension of previous leyer and dout is the dimension of this layer\n",
    "<br>\n",
    "Store in der_W gradient of the cost function with this layer's weights and in der_b gradient with biases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also store previous layer's activation and current z in forward pass in order to calculate gradients with deltas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In \n",
    "```python\n",
    "def backward(self, delta_n:np.ndarray, z_p:np.ndarray):\n",
    "    sd = sigmoid_der(z_p)\n",
    "    delta = (self.W.T @ delta_n) * sd\n",
    "    self.der_W += delta_n @ self.a.T\n",
    "    self.der_b += delta_n\n",
    "\n",
    "    return delta\n",
    "``` \n",
    "we pass next layers $\\delta$ as delta_n and previous layers $z$ as z_p\n",
    "<br>\n",
    "Then calculate $\\sigma'(z^l_j)$ with \n",
    "```python \n",
    "sigmoid_der(z_p) \n",
    "```\n",
    "and $\\delta = ((w)^T \\delta) \\odot \\sigma'(z)$ for previous layer and we'll pass in to the previous layers backward function.\n",
    "<br>\n",
    "$$\\delta^l = ((w^{l+1})^T \\delta^{l+1}) \\odot \\sigma'(z^l)$$\n",
    "here \n",
    "- $w^{l+1}$ = self.W \n",
    "- $\\delta^{l+1}$ = delta_n\n",
    "- $z^l$ is z_p\n",
    "We also accumulate sum of the calculated gradients in der_W (for weights) and der_b (for biases) in order to use it in mean calculation in batch for gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def zero_grad(self):\n",
    "    self.der_W = np.zeros(self.W.shape)\n",
    "    self.der_b = np.zeros(self.b.shape)\n",
    "```\n",
    "Cleans accumulated gradients for next batch processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run feed forward our linear layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.random.rand(28*28, 1)\n",
    "layer = Linear(28*28, 28*28 + 100)\n",
    "z = layer(x)\n",
    "a = sigmoid(z)\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can implement the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNetwork(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layers = list()\n",
    "        self.acts = list()\n",
    "        self.layer_dict = OrderedDict()\n",
    "        \n",
    "    def forward(self, *args, **kwargs):\n",
    "        pass\n",
    "    \n",
    "    def backward(self, x, y, out, **kwargs):\n",
    "        z_p = None,\n",
    "        delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "        for i in range(1, len(self.layers) + 1):\n",
    "            layer = self.layers[-i]\n",
    "            z_p = self.layers[-i - 1].z if i < len(self.layers) else x\n",
    "            delta = layer.backward(delta, z_p)\n",
    "            \n",
    "    def zero_grad(self):\n",
    "        for layer in self.layers:\n",
    "            layer.zero_grad()\n",
    "            \n",
    "    def save(self, filename:str):\n",
    "        weight_dict = dict()\n",
    "        for k, v in self.layer_dict.items():\n",
    "            weight_dict[k] = [v.W, v.b]\n",
    "        with open(filename, 'wb') as handle:\n",
    "            pickle.dump(weight_dict, handle)\n",
    "            \n",
    "    def load(self, filename:str):\n",
    "        with open(filename, 'rb') as handle:\n",
    "            weight_dict = pickle.load(handle)\n",
    "        for k, (W, b) in weight_dict.items():\n",
    "            if k in self.layer_dict:\n",
    "                self.layer_dict[k].W = W\n",
    "                self.layer_dict[k].b = b\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __setattr__(self, name, value):\n",
    "        super().__setattr__(name, value)\n",
    "        if not hasattr(self, 'layers'):\n",
    "            self.layers = []\n",
    "        if not hasattr(self, 'layer_dict'):\n",
    "            self.layer_dict = OrderedDict()\n",
    "        if isinstance(value, Linear):\n",
    "            self.layers.append(value)\n",
    "            self.layer_dict[name] = value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here\n",
    "```python\n",
    "def __setattr__\n",
    "```\n",
    "will store all linear layers in list for further automatic processing\n",
    "the method:\n",
    "```python\n",
    "def backward(self, x, y, out, **kwargs):\n",
    "    z_p = None,\n",
    "    delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "    for i in range(1, len(self.layers) + 1):\n",
    "        layer = self.layers[-i]\n",
    "        z_p = self.layers[-i - 1].z if i < len(self.layers) else x\n",
    "        delta = layer.backward(delta, z_p)\n",
    "```\n",
    "First in will calculate delta for cost function:\n",
    "$$\n",
    "\\frac{1}{2n} \\sum_x \\| y(x) - a\\|^2.\n",
    "$$\n",
    "<br>\n",
    "$$\n",
    "\\delta^L = \\nabla_a C \\odot \\sigma'(z^L).\n",
    "$$\n",
    "by\n",
    "```python\n",
    "delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "```\n",
    "Then layer by layer from end to beginning runs backpropagation calls each layers backward function passing the next layers delta_n ($\\delta^{l+1}$) and previous layers z_p ($z^{l-1}$) and calculates deltas on the fly, layers accumulate gradients sum for weights and biases as we implemented above\n",
    "<br>\n",
    "The method\n",
    "```python\n",
    "def zero_grad\n",
    "```\n",
    "Cleans gradients by weighs and biases for each layer for next batch processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Simple feed forward model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNModel(NNetwork):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = Linear(28*28, 100)\n",
    "        self.fc2 = Linear(100, 50)\n",
    "        self.fc3 = Linear(50, 10)\n",
    "        \n",
    "    def forward(self, x:np.ndarray):\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = sigmoid(z1)\n",
    "        z2 = self.fc2(a1)\n",
    "        a2 = sigmoid(z2)\n",
    "        z3 = self.fc3(a2)\n",
    "        out = sigmoid(z3)\n",
    "        \n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layer_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run our model on randomly generated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model(x)\n",
    "out = np.argmax(pred)\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize random label and test backward (backpropagation) and zero_grad methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0]).reshape(10, 1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.backward(x, y, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ll in model.layers:\n",
    "    print(ll.der_W.shape, ll.der_b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.zero_grad()\n",
    "for ll in model.layers:\n",
    "    print(np.count_nonzero(ll.der_W), np.count_nonzero(ll.der_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to implement the loss function and gradient descent algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD(object):\n",
    "    \n",
    "    def __init__(self, lr):\n",
    "        self.lr = lr\n",
    "        \n",
    "    def optimize(self, model, batch):\n",
    "        for x, y in batch:\n",
    "            y_hat = model(x)\n",
    "            model.backward(x, y, y_hat)\n",
    "        for layer in model.layers:\n",
    "            layer.W = layer.W - self.lr * (layer.der_W / len(batch))\n",
    "        model.zero_grad()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The stochastic gradient descent will calculate mean of gradients in batch then makes step for gradient descent with learning rate and cleans accumulated gradient s for next batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = SGD(0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After each epoch we'll run validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(test_data):\n",
    "    y_hats = [(np.argmax(model(x)), np.argmax(y)) for (x, y) in test_data]\n",
    "    valid_res = sum(int(x == y) for (x, y) in y_hats)\n",
    "    \n",
    "    return valid_res\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we call model (with updated weights and biases) and compare it's output with labels and count correct predictions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, training_data, test_data, epochs:int=12, batch_size:int=16):\n",
    "        if test_data: n_test = len(test_data)\n",
    "        n = len(training_data)\n",
    "        for j in range(epochs):\n",
    "            random.shuffle(training_data)\n",
    "            batches = [training_data[k:k + batch_size] for k in range(0, n, batch_size)]\n",
    "            for batch in batches:\n",
    "                 optimizer.optimize(model, batch)       \n",
    "            if test_data:\n",
    "                valid_res = validate(test_data)\n",
    "                print(f'Epoch {j}: {valid_res} / {n_test}, accuracy = {valid_res / n_test}')\n",
    "            else:\n",
    "                print(f'Epoch {j} complete')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The training methods will iterate over the epochs, shuffle before each epoch data, group them in batches, run gradient descent per batch and validate model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train(model, optimizer, training_dt, test_dt, epochs=12, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serialize the Trained Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data') / 'weights.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(str(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dsk = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_1 = model_dsk.layers[0].W.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dsk.load(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_2 = model_dsk.layers[0].W.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.all(W_1 == W_2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}