Exercise2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The objective of this exercise is to implement a multi-layer perceptron with one hidden layer from scratch and test it on MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets\n",
    "import os\n",
    "from dlc_practical_prologue import load_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigma(x):\n",
    "    return torch.tanh(x)\n",
    "\n",
    "\n",
    "def dsigma(x):\n",
    "    return 1-(sigma(x)**2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(x,t): \n",
    "    return torch.pow(torch.norm(x-t),2)\n",
    "      \n",
    "def dloss(x,t):\n",
    "    return 2*(x-t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input, train_target, test_input, test_target=load_data(one_hot_labels=True, normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forward and Backward Pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x=train_input[0].unsqueeze(1)\n",
    "# s1 = torch.mm(x.T,w1) + b1\n",
    "# print(w1)\n",
    "# x1 = sigma(s1)\n",
    "# s2 = torch.mm(x1, w2) + b2\n",
    "# x2 = sigma(s2)\n",
    "# x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_pass(w1,b1,w2,b2,x):\n",
    "    s1 = torch.mv(w1,x) + b1.flatten()\n",
    "    x1 = sigma(s1)\n",
    "    s2 = torch.mv(w2,x1.T) + b2.flatten()\n",
    "    x2 = sigma(s2)\n",
    "    return x,s1,x1,s2,x2\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_pass(w1,b1,w2,b2,t,x,s1,x1,s2,x2,dl_dw1,dl_db1,dl_dw2,dl_db2):\n",
    "    \n",
    "    dl_dx2 =dloss(x2,t) \n",
    "    dl_ds2 =dl_dx2*dsigma(s2)\n",
    "    dl_dw2 =torch.mm(dl_ds2.view(-1,1),x1.view(1,-1))\n",
    "    dl_db2 = dl_ds2\n",
    "    dl_dx1 = w2.t().mv(dl_ds2)\n",
    "    dl_ds1 = dl_dx1*dsigma(s1)\n",
    "    dl_db1 =dl_ds1\n",
    "    dl_dw1 =torch.mm(dl_ds1.view(-1,1),x.view(1,-1))\n",
    "    \n",
    "    return dl_db1,dl_db2,dl_dw1,dl_dw2\n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_target = train_target * 0.9\n",
    "test_target = test_target * 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden=50\n",
    "epsilon=1e-6\n",
    "n_classes=train_target.size(1)\n",
    "n_train_samples=train_input.size(0)\n",
    "eta=0.1/n_train_samples\n",
    "\n",
    "\n",
    "w1=torch.empty(n_hidden, train_input.size(1)).normal_(0,epsilon)\n",
    "w2=torch.empty(n_classes, n_hidden).normal_(0,epsilon)\n",
    "b1=torch.empty(n_hidden).normal_(0,epsilon)\n",
    "b2=torch.empty(n_classes).normal_(0,epsilon)\n",
    "# w1 = torch.normal(0.0,1e-6, size = (train_input.size(1),50))\n",
    "# b1 = torch.normal(0.0,1e-6,size = (1,50))\n",
    "# w2 = torch.normal(0.0,1e-6,size = (50,10))\n",
    "# b2 = torch.normal(0.0,1e-6,size = (1,10))\n",
    "\n",
    "dl_dw1 = torch.empty(w1.size())\n",
    "dl_db1 = torch.empty(b1.size())\n",
    "dl_dw2 = torch.empty(w2.size())\n",
    "dl_db2 = torch.empty(b2.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in range(1000):\n",
    "\n",
    "    # Back-prop\n",
    "\n",
    "    acc_loss = 0\n",
    "    nb_train_errors = 0\n",
    "\n",
    "    dl_dw1.zero_()\n",
    "    dl_db1.zero_() \n",
    "    dl_dw2.zero_()\n",
    "    dl_db2.zero_()\n",
    "\n",
    "    for n in range(n_train_samples):\n",
    "        x0, s1, x1, s2, x2 = forward_pass(w1, b1, w2, b2, train_input[n])\n",
    "\n",
    "        pred = x2.argmax()\n",
    "        if train_target[n, pred] < 0.5: nb_train_errors = nb_train_errors + 1\n",
    "        acc_loss = acc_loss + loss(x2, train_target[n])\n",
    "\n",
    "        m1,m2,m3,m4=backward_pass(w1, b1, w2, b2,\n",
    "                      train_target[n],\n",
    "                      x0, s1, x1, s2, x2,\n",
    "                      dl_dw1, dl_db1, dl_dw2, dl_db2)\n",
    "    \n",
    "    dl_db1 +=m1\n",
    "    dl_db2 +=m2\n",
    "    dl_dw1 +=m3\n",
    "    dl_dw2 +=m4\n",
    "    \n",
    "        \n",
    "\n",
    "    # Gradient step\n",
    "\n",
    "    w1 = w1 - eta * dl_dw1\n",
    "    b1 = b1 - eta * dl_db1\n",
    "    w2 = w2 - eta * dl_dw2\n",
    "    b2 = b2 - eta * dl_db2\n",
    "\n",
    "    # Test error\n",
    "\n",
    "    nb_test_errors = 0\n",
    "\n",
    "    for n in range(test_input.size(0)):\n",
    "        _, _, _, _, x2 = forward_pass(w1, b1, w2, b2, test_input[n])\n",
    "\n",
    "        pred = x2.argmax()\n",
    "        if test_target[n, pred] < 0.5: nb_test_errors = nb_test_errors + 1\n",
    "            \n",
    "    if k%20==0:  \n",
    "        print('{:d} acc_train_loss {:.02f} acc_train_error {:.02f}% test_error {:.02f}%'\n",
    "          .format(k,\n",
    "                  acc_loss,\n",
    "                  (100 * nb_train_errors) / train_input.size(0),\n",
    "                  (100 * nb_test_errors) / test_input.size(0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forward_pass(w1,b1,w2,b2,train_input[0]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,s1,x1,s2,x2 = forward_pass(w1,b1,w2,b2,train_input[0]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t=train_target[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#backward_pass(w1,b1,w2,b2,t,x,s1,x1,s2,x2,dl_dw1,dl_db1,dl_dw2,dl_db2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl_dx2 =dloss(x2,train_target)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}