02 - 1. MNIST Feedforward Neural Network(more layers).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import PyTorch\n",
    "* 即時顯示圖片\n",
    "* 載入`torch`模組\n",
    "* 載入`torchvision`pytorch中有關圖形處理的模組\n",
    "* 載入pytorch中有關資料集的模組\n",
    "* 載入pytorch中的data transform模組(不一定是augmentation)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Hyper-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_size = 784       # The image size = 28 x 28 = 784\n",
    "hidden_size = 520      # The number of nodes at the hidden layer\n",
    "num_classes = 10       # The number of output classes. In this case, from 0 to 9\n",
    "num_epochs = 10         # The number of times entire dataset is trained\n",
    "batch_size = 100       # The size of input data took for one iteration\n",
    "learning_rate = 0.001  # The speed of convergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = datasets.MNIST(root='./data',\n",
    "                           train=True,\n",
    "                           download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transforms.ToTensor(train_dataset)\n",
    "# transforms.ToTensor()(train_dataset)\n",
    "# transforms.ToTensor()(train_dataset[0][0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = datasets.MNIST(root='./data',\n",
    "                           train=True,\n",
    "                           transform=transforms.ToTensor(),\n",
    "                           download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data',\n",
    "                           train=False,\n",
    "                           transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset MNIST\n",
      "    Number of datapoints: 60000\n",
      "    Root location: ./data\n",
      "    Split: Train\n",
      "<class 'torchvision.datasets.mnist.MNIST'>\n",
      "60000\n",
      "torch.Size([1, 28, 28])\n",
      "5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA5klEQVR4nGNgoD9gRGJr+aSevsAw4Rc2demf/v379++fE1ZDhF78+/fv3793blhlM778e/Dv379e7A44/+/Sv3//lLBLhpz79+/fP00cbpe4+O/fv9VwLguyXLSeDgMDw1Fs2jSu/fr3D8VOJoSkpiLEmAKsNuZ9+/cPp52TbgswsEzmw+FYBgYGxoZ/t+VxSbL/+3dNBpdk179/JehiwhujGBgYGBgkP2AJviX/rturMBhHnvv3r5sDXdLy6L9/97Z8/Pfv71VuTLt6Mv/9+/fv3783yIIwf5aw8zAYRDJ8xB7TdAQABFdhZWAfWxoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=28x28 at 0x7FEB65904048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(train_dataset)\n",
    "print(type(train_dataset))\n",
    "print(len(train_dataset))\n",
    "# print(train_dataset[0].shape)\n",
    "print(train_dataset[0][0].shape)\n",
    "print(train_dataset[0][1])\n",
    "\n",
    "img = transforms.ToPILImage()(train_dataset[10][0])\n",
    "display(img)\n",
    "print(train_dataset[10][1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We shuffle the loading process of train_dataset to make the learning process independent of data orderness, but the order of test_loader remains to examine whether we can handle unspecified bias order of inputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=True,\n",
    "                                          num_workers=4, \n",
    "                                          pin_memory=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False,\n",
    "                                          num_workers=4, \n",
    "                                          pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7febe11495c0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAToAAAD8CAYAAADnhGhBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXm8TeX6wL+PMVwylCGUBnGlFCndJpEMSZo0uamE0qh08aMyVJTSKKLxlgY0kIqMud0rhUIoGnWiJFGhQu/vj7WftfY5Z59z9jl73p7v57M/++y13r3Xc9Ze+13P+4zinMMwDCObKZVqAQzDMBKNTXSGYWQ9NtEZhpH12ERnGEbWYxOdYRhZj010hmFkPTbRGYaR9SRsohORDiLymYh8LiIDE3UcwzCMopBEBAyLSGlgLdAOyAE+BC52zq2O+8EMwzCKoEyCPvc44HPn3JcAIvIScDYQcaITEUvPMAyjJGx2zu1f1KBELV3rAt+Gvc4JbfMRkd4iskREliRIBsMwsp9vohmUKI1OImzLpbU55yYAE8A0OsMwEkuiNLocoH7Y63rAhgQdyzAMo1ASNdF9CDQUkYNFpBxwETA9QccyDMMolIQsXZ1zu0XkOmAWUBp4yjm3KhHHMgzDKIqEhJcUWwiz0RmGUTKWOueOLWqQZUYYhpH12ERnGEbWYxOdYRhZj010hmFkPTbRGYaR9dhEZxhG1pOoFDAjTalUqRIACxYs4IADDgDgxBNPBODrr79OlVhGFtGsWTMAJk+eDMDhhx8OgHOOn376Kde2n3/+OSky2USXhegEtv/+QVEHvaBOO+00AFq0aMFnn30G4F98RuKpWrUqAJ06dQKgRo0aAEyYMIE//vgjZXKVlFKlvEXhUUcdxTPPPANAw4YNAahQoQIAf/31lz9e/99XX30VgC5dugDw66+/JlbOhH66YRhGGpDRGp2IVyRl3LhxAPTu3RuALVu28P333wOwapWXeabaSzh657nwwguLPNYHH3wAwCmnnALAn3/+GYvoMXPkkUdy/fXXA3DQQQfl2qfLggMPPNDfNmrUKACaNGkCeOfuu+++A6BcuXIJlzcaLr30UgAOPfRQAL788ku6du0KwLnnnlvk+/V60GyfMWPGANC/f/+4y1ocunfvznnnnQfA6aefDkDFihVzjenQoQM333wzEPlaTVcuueQSAJ599ll/m57/F198EQg0Oh0Lwe/o3XffBbxzBLB6dWJq85pGZxhG1pPRua663t+0aVNc5YnErl27ch1z+/btCT9mYdxwww088MADEfeprWfKlCm0bdsWgDp16uQaIyJcdtllADz//PMJlLRo6tb1arJ+9NFHQHCOY+Wrr74C4LDDDovL50WLash6Xi+44AKK+p2JCD/++CMA7du3B4LzkY6ojXH6dK8oUalSpfj0008B6NGjBwAffvghAIMHDwZg+PDh/ni1zakWPn78eAD69u1bXFEs19UwDAMy3EZ3xhlnFDlGvTl6R1XbyM6dO9lnn31yjdl3330BKF26dL7Pue+++4DUa3JDhw4F4NZbb/W3qX1ENQKV9ccff+Too48GYNasWQDst99+/r6pU6cmRebCKFOmDE8++SRQPE3ut99+A7zvUUNm8tq99PtNNg0aNADw7XLOOf/6y8nJAeDpp58GYPPmzQDcfvvt/v+v9lR9v/6v6cTZZ58NBBoZeJ5XgN27dwNQvXp1APr06ZPvfXk13K1btyZOWDJ0otMLeMCAAQWOWbt2LRCo2Dt37gTg+OOPB2DZsmUceeSRALz11lsArFy5EggM9gALFy4E4M4774yb/LGgP+oKFSrwzTdeuXxdGmzcuDHX2MMOO4z/+7//A4JQkx07dgAwbNgwfv/996TIXBh33nkn7dq1i7hv7dq1XHfddRH3/fDDD4D3A7nxxhsBfGO+cvfdd8dR0ujRa01vir/99htPPfUUABMnTgTwvzulSZMm/oSgDgu9VufOnZt4oaOkTBlvymjcuHGu7S+//DJ79uwBoHLlygBcc801QGCaCEfP0fDhwwF46KGHEiNwCFu6GoaR9WSkRqdLS10iKKoyr1y5km7dugGBQVqZNm2a//e333qNypo3bw4Egbbh3HPPPQBpof0A/nKzY8eO/P3vfweCpY4acnUJPmbMGM4880zAC7kBuOuuuwB47LHHkid0BI477jiAXBqbLmfuvfdewNPIilq2NW7c2A+zUTQ4WjX1ZKPXVcuWLQEvIFuXqAWhK5Bw1DSTThqdamcnnXRSru21atXi6quvBoLvVK/P8GWqZkvoNfvxxx8nVuAQptEZhpH1ZKRGp7YPdVFPmDABgJdeegkIDPbRcsMNNwBBeg7Aa6+9BgQBjemC3gEXLVrk3zE1hERtXRp2Eh4wPGzYMAAeeeSRpMkaCdWa33jjDSBIEwLPzgP4dsVoPmfq1KmULVsWCIK41eCd6txdladDhw5MmTIFoMA0rxYtWuTb9sorryROuBKiYVZq61UH0Kmnnsqpp56aa6yOmTFjBuD9pvQ8hKeFJQPT6AzDyHoyUqNT1COa1wMULeodUltKOOrBU+9QuqAawS+//OJv02Bg1QDCU6E0dOP1119PppgForYdDXOBQO6BAwcW+X61z95yyy1AYAeCIAXwv//9b3yELSadO3cGvFARgGOP9eJYRYR///vfQBBgq7ZW9fx3797dt2UtW7YM8FLg0o0NG7z2zP/73/+AwEMsIr78+j/ecccdAKxYsSLZYubDNDrDMLKejNboYkWT+UuqEaaSvHFYkXjrrbf84GH1BKaaK664Itfr1atX069fPwC/yEBhXHvttQDcdNNN/ja1AaUyALpPnz6+/VO1Tk2JEhG/0MJZZ52V61kJDyrOG0ycTqiHW+3CinPOj4nT53RIL1X22omuatWqfqBpXubNm+dnGaQb+iM6+eSTc0Wlh/Pmm28C+X9M6YCe844dOwIwadKkqH7QF110EQBDhgzJtT0nJ8evepIKM4MuV0eOHOl/N2PHjgVg0KBBgBe4rU4tNZcUhponNDA+1aFN+n+NHj3av9FEuvZGjhwJpNcEp9jS1TCMrCejq5fEQrt27Zg5c2bEfR06dGD27NlJlig61D1fWH021eg0/CaT0aBwrQeo+aAaAN2xY0eWLFmSEtnC5WrRogXz588H8ANnNQWqX79+UVUvyTtGaynOmDHDd1Dodblt27Y4/QdFo4HXHTp08GXU8CB1phx88MH+krW44V0xktjqJSJSX0Tmi8gaEVklIjeGtlcXkdkisi70XK2kxzAMw4gHsdjodgO3OOeWiUhlYKmIzAYuB+Y650aJyEBgIFBw9n0aMW/ePADee++9FEsSoIGxasQPr4ihd/nly5fnGlOzZs1ki5kQypYt62uneSubPPzwwwAp1eYgCCFxztGoUSOg8ArBecNLtF/HP/7xD78ijdazO+KII/xntYlpeptqdqpZtW7dml69esXpv/LQOn4afuWc45133gGCatDnn38+AE899ZS/ghgxYgSAn+SfDpRYo3PObXTOLQv9/SuwBqgLnA1oXeVnga6xCmkYhhELcfG6ikgD4BhgMVDLObcRvMlQRNJSvbj88svzbVNPazoFCasbX+0fypAhQ3j00UcB/L4KqtElqu5+sjnrrLPyhf488cQTQFCcINVoAO0BBxwQsRwRwMyZM/0KuqrRRRqj2pmmg6nX/IADDvDrClar5lmCtGiFPqsc8aB8+fIAzJkzBwjqym3evNn3emsa5uLFi/33aZtDrUuXThWSY57oRORvwCvATc65XwoKeYjwvt5A71iPbxiGURQxTXQiUhZvkpvknHs1tPkHEakT0ubqABEbOjjnJgATQp+TNK+rdiKKFGMWqVROKmndurVvi1LUDjJnzhxq164NBClHSqqT2WNF7T7aJxSCdCi1/yQ7Kbwg2rRpA+Sudq3J/GqHUw9xUWiPBX1WLbB8+fJ+wVW1CWoAsh7rhRdeKPk/kQctrhleFELlWbp0aa5tX3zxBeAVFNUUPC17lhUanXiq25PAGufcmLBd04EewKjQ87QIb08ZumTVCweCiHxdFqUL7dq182vLacCpZgGULVvWD1bVMapNp2NEfTSoE0UrrVSoUMH/IesEp6XI0wW9OSbyJvnHH3/4QcTqDNDnRJI33EWdX+FoNZNFixb5zhDNWdY863QgFo3uROCfwEoR0ep5/4c3wU0WkZ7AeuCC2EQ0DMOIjRJPdM6594CCDHJtC9iedNRF/+qr3so6ksFY6/mnm7YQnv+oz1p7rWvXrn6dfQ05UI001dWDS4pqcuqAcM75KUda/cNIPGoW0OrOmrZWpUqVAt/z1ltv+eEkeZe86YClgBmGkfVkfVJ/q1atgMhNjLWih2p06YZ27oIg9EUDRU8++WR/n4aVaHhCpqHVOjQIVQnvnmUkDw2v0uB0rRc4ZswYP+VLw2S0c97w4cP95H/t3ZJOmEZnGEbWk/UanXbBioSGL6xfvz5J0hSPNWvW+H9ryIV6Vrds2eKXA9LAzkzkqKOO8kMzVCPQZsaqURipQa+rf/3rX4DXEa9///5AUA9QtbfwZuH3339/MsWMiqyd6LRU96GHHhpx//r16/0lU7ry7LPP+nmPt912GxDkdk6fPt1vgpPJjBgxwo8HVDSMYcGCBSmQyMjL448/DsD8+fP90unnnHMOEMQRvvHGG77DL5UFUAvClq6GYWQ9WVuPrmnTpkBQ2SMvc+bMoX379vE+rFFMNm3a5FcmUSO4fneZnuFhJIXE1qMzDMPIFLLWRqfhGFobTGuFqeNBq78a6cPgwYMB0+SM+GManWEYWU/W2ugMw9grMBudYRgG2ERnGMZegE10hmFkPTbRGYaR9dhEZxhG1mMTnWEYWY9NdIZhZD020RmGkfXYRGcYRtZjE51hGFmPTXRZSPfu3enevTvbt2+nfv361K9fP9UiGUZKydrqJUUxcuRIv0S00q9fP8Cr7Ltt27ZUiBUTp556KgDjxo0DvAbQxx13HBA0uV61ahWQvzmxkRyGDh2a67V+Z61bt/YrKmuz8rxj05kuXbowbZrXq17bJSrafnP37t1+fcjFixcDudsFJBLT6AzDyHr2muolTZo0AeD1118HoH79+n4z6DA5AHj77bf9FoJa1y4TePbZZwH45z//WeCYF154AYA+ffqwffv2pMi1t9O6dWvmz59f7PephnfaaafFWaL48/nnn3PwwQcD0a0W9NrTlUYMWPUSwzAMyGIbXaVKlQAYNmwYEHQtatCgAVD4XadDhw7MmDEDCDogpXMj5datWwNw7rnn5tv3yy+/AFC5cmUALrnkEsDTFp544onkCFgINWrUoGLFigA0b94cgFNOOcXf37BhQyBoW1mqlHdv1mbdI0aM4Pvvvwdgz549AGzYsCEJkkePfj8lfd/QoUPTzl5XvXp1AG655RYA6tatW6z3V6hQAQiuWe0glihi1uhEpLSIfCQiM0KvDxaRxSKyTkReFpFysYtpGIZRcuKh0d0IrAGqhF7fAzzgnHtJRMYDPYFxcThOsVCtQBvtFpcWLVoAMGHCBACaNWsGeBoEwObNm2MVMWZq1aoFwM033wwEWux3330HwN133+33rtVm3d26dQOgfPnyyRTVp3PnzgBcdtllABx//PH5tAG1lYZr3fq3evQ6deqU6xmCnretWrVKhOglRj2r2cRhhx0GwIABA/Lt+/LLLwH8a++EE07wn6tVqwYE2vcXX3yRcFkhxolOROoBZwJ3ATeLd4W2AS4JDXkWGEqSJjr9gVx33XV+o5V4cd111wFwyCGHAHDWWWfF9fOLS61atZg4cSIQTB6KTnxTpkzxt23atCnXmM6dOzN27NgES5kfnZjOO+88ILIJQX8gb775pr9fl6fXX389ABdeeGG+96l5It2ItHRVR0NJl7WpQk0/zz//fL59K1euBIIwrbwOmP32289vgL17926g4Hak8SbWpeuDwL8ADZypAWx1zu0Ovc4Bird4NwzDiDMl1uhEpDOwyTm3VERa6+YIQyNa/UWkN9C7pMePxN///ncAHnjggRK9/+233wa8Za8ayPOiy5A+ffr4jopkUqaM95VNmjSJNm3a5No3c+ZMAN+RUhjNmjVjn332AeD333+Ps5T5Ofzww4FAE/vtt98AzzTw8ssvA4HWlpOTk+/9vXr1AvJrr4Cv2W7cuDHOUieOvCEj8+fPzwjtTp1ZurIJZ+DAgUB+TU7ZvHkzL730UuKEK4RYlq4nAl1EpBOwD56N7kGgqoiUCWl19YCILjDn3ARgAlgXMMMwEkuJJzrn3CBgEEBIo+vvnLtURKYA5wMvAT2AaXGQs1DKlfMcu3lTuiLx+eefAzB8+HDOP/98IEiLGjJkCOBpdKNHjwYCp4Simt5NN92UEo2ue/fuALm0OW3Krba5nTt3Fvk5v/zyS75UnURy9NFHA1C1alUgCBidNm2a70SIhJ5vDa/QMBm13e3cuTMqDdaID1dddVXE7X369PFXFOlIIgKGB+A5Jj7Hs9k9mYBjGIZhRE1cAoadcwuABaG/vwSOi8fnRotqMqrtRGLq1KkAXHTRRf62SZMmRRy7cOFCX1vT8JJ0QQM1IdDkOnToAMCnn34a9ef8/vvvSU3s13CcP/74AwhCYV599VWuvvpqwPOyho+pXLmy74HVUBqVWZ+vueYa/32ZiNqzMsE+16lTJ2rWrJlr27x58wCYPHlyKkSKGksBMwwj68noFDCN6VFNTuPowtm6dSsQlIqJFk1J0QR5veOqXUtTkZKNaqZly5b1k/jVW1kYVapUyfX60EMP9Ysa7Nq1K85S5kfv/Bq43KdPH8DTUFUbePJJz8px//33A9C3b1+6du0K4Hvrjj3Wy9/WgNV169YlXPZEEI02rbF26ZL+NWTIEN9Tr4waNQqAX3/9NRUiRU1GVy/RPLt77rkn3z6tOnL55ZcDMGvWrGJ9dpcuXYBgwssbrb9u3To/nCVVaG6oTgbKJ598AniVWnQy+/rrrwEvtxRgzJgx9O/fP0mS5ufSSy8FvOVQ3uBfzc8tW7Ys3377LQBnnHEGEDiHdKJ75JFH+PPPP5Mic0kZOnQod9xxR7HfF+nGnUpat27t5xhrrqo6AMeMGZMqsax6iWEYBmTo0vXss88GKPQuqcG/xdXkFF1a5UXDIkoalBwrJ510EgD33Xefn39bUN7qsmXLeOedd4BAk1OK47hIBOoImjRpkl9hZs6cOQDUq1cP8LRnDQfS5blWsc1G0r3+3IIFC/yg7EMPPRTAD8PavXu3b0pJx+rcptEZhpH1ZKRGd+CBBwIUmKYFcNdddyXk2AsXLgSSH3aiVUfUqVK9enW/7trs2bNzjdUQmubNm/t2vLxs2bIlUaIWm7Vr1wKBlnzfffcBnkanRQA0mFvPe7rVnIuEOhGisc8tWLAgbTW5cNQerCulOnXqAJ6NTmsHau3Gjz76KAUSRsY0OsMwsp6M1OjUG1WQV2r58uUx2QlGjhzpB+EqGk6SbE+YlrwZOXIkEATeXnbZZb4HLO//qvXA1PYVjmqE2jsjHdEUthkzZvipbqrR6feSbjXnwimOJqdE+q7SkdWrVwNBSNfcuXP9fX379gWCWoM9evQA0uNaM43OMIysJyM1urxpQHl59913S1QBWLWmm266Kd9nf/XVVwAMGjSo2J9bUlq3bu3LlDeeTOWJRGH2Kw3KTWZCf7SoPfGnn34CPFvjDTfcAHhFGMLHvP/++4BXbDNdyjOVRJNT5s+fnzZxcy1btgSC6Abl8ccf96/D//znP0BQrGHq1Km+jU7T+zTIW4OKR40alZSSYJHIyImuKKKd5LSEtxq4dVkUaQLV5izJaLirzpaJEyf6lVneeustoPAJTieBu+++u8AxGgidToZidSrpMjV8Kf7www8DwY9PA4319UEHHZQ2E100JdN1iRreuFrRvzXMJFW0b98eCOrLKccee6zvFNOgbg1Ob9y4sW8W0TFaqUbNDkcccYTfRlTrESYLW7oahpH1ZKVGF00w7G233eYHPeodLBKaW5oMTU5p27Yt4AVlasBvYUtmrd6iITUaQLxt2zY/1/eggw4C8HtpPPjgg2kT2KnaslYo0fCScLQp0Y4dOwDo2bMn4OUiq3aqVU9SwdChQwusQBIeOqJjSrK8TRYFmYROP/1039Hw6KOP5tuvzoj33nsPgH//+9+59p9zzjn+d7t48eK4yRsNptEZhpH1ZKRGV1R4yeTJk30NQA3b2ig3bxWScDSE5K+//vKN9r17x7WtRbHRUBHVZDSZum/fvn6FZLVXqfxqqO/bt6/f/UvTqxo3bgx4refSpSLsMcccU+QYDSq+8847gUCj69Onj691F9SrIBkUpaFlUt05DQbWlUV4Q3G1/+rKQIteaItNiNxIXdEKxabRGYZhxJmM1OiKCi+BwNOTF9XkIr1XPZpTp05NaQ2wDz74wP9bu1+1a9cOgPr16wNBnwwIaoGpjU5L5mjvTAgacKutLlmNg6MhVg+wpomlUqNbsGBBgdpaNFrcsGHDUu5tVbSHxwUXXADAokWLAK/zl3rI9Tm8+EWkxuN50RVJssnIiW769OlA0PpPJyWN34kWjTdTo6lW1Eim4yESGn3eoUMHnnvuOSCoFqFs376dBx98EAgaVa9YsaLAz9TimloNJJ3QTAi9CWmYyeOPP+4v2bUpzm233QYEP6qdO3emshaaz2mnnVai0vTpVlwzHDX7qGlk1KhRfrZD+I02WpYtW+bnwSYbW7oahpH1ZHSFYeXKK68EoqsooprAiBEj/MYr33zzTSyHN+KEfg9aj27RokW+kVuj7nUJrtftFVdc4Wu9qUaXqNGEkGi4SbosV6NFVxa33347EARwf/zxx74JJa8pYty4cYDXYDwBgcJWYdgwDAOyRKMzsgMNPdBWk5GuTdXIX3nlFcALM0n3xixGQjGNzjAMAzLU62pkJ0888QQApUuXBmDs2LH+vmXLlgFBcQPt/KYeW8MoDNPoDMPIesxGZxhGJpN4G52IVBWRqSLyqYisEZETRKS6iMwWkXWh52qxHMMwDCNWYl26PgTMdM41BpoBa4CBwFznXENgbui1YRhGyijx0lVEqgDLgUNc2IeIyGdAa+fcRhGpAyxwzjUq4rNs6WoYRklI+NL1EOBH4GkR+UhEnhCRSkAt59xGgNBzzUhvFpHeIrJERJbEIINhGEaRxDLRlQGaA+Occ8cA2ynGMtU5N8E5d2w0s7FhGEYsxDLR5QA5zjmtoDcVb+L7IbRkJfS8KTYRDcMwYqPEE51z7nvgWxFR+1tbYDUwHegR2tYDmBaThIZhGDESa2bE9cAkESkHfAlcgTd5ThaRnsB64IIYj2EYhhETFjBsGEYmY0n9RvZSunRpSpcuzbp163DO4Zyjdu3a1K5dO9WiGWmITXSGYWQ9Vr3EyCiaNm0KBG0PDznkEL91ntWlMwrCNDrDMLIe0+iMjODYYz17s/YqOPPMM/19qtFt3749+YIZPh07dgSCptbly5cHYOnSpUDQTSwVmEZnGEb2ox6rVD4Al4pHrVq1XK1atdwbb7zh3njjDbd+/Xq3fv36lMhij8iPli1bupYtW7qNGze6jRs3uj179uR63H777a5y5cqucuXKKZd1b35cc801bufOnW7nzp35vqNdu3a5Xbt2ubZt2ybi2EuimWNMozMMI+vZq210aks4/vjjAfweopmK9lo47LDDAOjWrRunn346AP369QOC3guZwujRowGoWdMrgrNjxw4A7rrrLgAee+yxrPW2NmjQAICLL76YSy65BIAmTZoA8McffwDQtm1bwOuBmwrUVnrvvfeye/duAF/WU089FYDrr78egBYtWjB37twUSLkXT3Q1atSgevXqqRajxAwcONBvgqyUKeN9nXm3A9xyyy1A0HA4E2jcuDGNGuUuZThr1iwARo0alQqR4k6LFi0AaNWqlb/tpptuAvCDnytUqODvy5vJ9Le//S3RIkZEb6AvvPACABUrVuTaa68F4LXXXgNgn332AYKJTkODUoEtXQ3DyHr2Wo3u8MMPp2HDhrm2ZdKyrk+fPtSvXx+AUqW8+5UuHSLx448/JkWueFCxYkUA+vfv7y9Z05XGjRsDXkPtJ598EoDNmzcXOP6ss84CAnNJlSpVgOJrZr///jsAs2fPLp7AMbLffvsBgUatcs+fP5/x48fnGrtkSe6aupFWGsnCNDrDMLKevVaj69evHyKSa5s2R84EDj74YFq3bg3g27GmT58O5HaqqCandp90RjW5hx56CIArrrgileJEhToDGjVqxL333lvs9+s1GG0VoR9++AHA/+6TzWWXXQZA8+bNAfjtt99ybQ9H7cH6Py5YsCAJEkbGNDrDMLKevU6jq1GjBuDZSPQuunHjRgAmTpyYMrlKgt4h9blatfwtdDWEJhNQm6OGUGzbto19990XgE8++QSAp556KjXCFcAZZ5wR0/s13GLYsGH8+eefAOy///6AFzoDUK9ePX+82mHXrVsX03FLitrd5syZA+DbJTds2OCP0bCYPn36AIG2mko78V430enyqG7duv62l156KVXixJXwEAVFlxa6HDzvvPMAuP/++3nkkUeAwp0YyeSzzz4DoFevXgC8++67/r7evXsDQV5ruvD8888DweQE8MUXXwDBku39999n9erVEd8/b968fNvUBBE+wSmpnugXLlwIFD7B16pVCwhiH3fu3AkEE3cqsKWrYRhZz16n0fXs2TPVIsQN1U41ZOGZZ57JN0YDhfMyatQoli9fDkTWKlKBhloMHToUICMCuqdMmZLruaRUq1aNBx54AIBTTjkl334N50i1RhcNeYO8VaP7/PPPUyEOYBqdYRh7AXudRnfHHXcA8Ndff/l3mjFjxqRSpBIxbNgwunTpAkCzZs0KHKc5ka+//joAL7/8MuCFoHz44YcJljI6KleuDMATTzwBBHbELVu2+ClGq1atSo1wSeKSSy6he/fuEfd9/PHHPP744wB8++23yRSrRGham/LVV1+lSJIA0+gMw8h69hqNrnPnzoCnyYHn8p4xYwYQhJdkAvp/DB482E/9yoveQc8++2ztEBB/AAAXTElEQVT/f/vpp5+SI2AJaN++PRBocsrUqVO58cYbAbjqqqsAuOaaa3KNeeyxx3jxxReBoLJJJqF2SdXOw1EvdNeuXcnJyUmqXCVBPc8azKzfxz//+c9UieRjGp1hGFnPXqPR5U3gh8yMn1uxYgXgxZgdeeSRQJBorXdQLaGTDraRoujUqVO+QG3V0G699VZ/m3rLjz766FxjJ0yYwCGHHAJ4Wm6moFqsph1GSgFTD2smaHMAHTp0AIJyTO+//z4QaKapZK+Z6CLx/fffp1qEYrN+/XrAy7HUH8Lll18O4EfWZ8IEp41UXnjhBd8ZoRkeWmyzUqVKfnjMUUcdVeBnhQfrpjsaCqTOITU/qEkFgiKpDz/8cJKlKzlVqlTxvyutrDJy5MhUipSLmJauItJPRFaJyCci8qKI7CMiB4vIYhFZJyIvi0i5eAlrGIZREkqs0YlIXeAGoIlzbqeITAYuAjoBDzjnXhKR8UBPYFxcpI0BvUtGuoNmIpUqVcpXLViXCumMpgVpmEulSpX4+eefAa/+HASVdceNG+fXbSuMjz/+OBGixo1q1apx9dVXA0G7Rl2qhjvH1PEyblzKfy5Ro7+ngQMH+lq3miLU2ZcOxOqMKANUEJEyQEVgI9AGmBra/yzQNcZjGIZhxESJNTrn3Hcich+wHtgJvAMsBbY65zRLPAeoW8BHJBW9u2hFhc8++yxlFSDiRdmyZYGgoq0GQ6czqqFVqlTJ36ZJ7B999BEQfFfRaHODBg3yg2nTlY4dOzJixIhCx9x4441+hd50W220adMG8DRu1UTVKaavBwwY4FcwKSjtMJWUWKMTkWrA2cDBwAFAJaBjhKERKwqKSG8RWSIiSyLtNwzDiBexeF1PB75yzv0IICKvAv8AqopImZBWVw/YEOnNzrkJwITQe6MrrxoDebW3LVu2pHUQbVEMHz7c/1trmqVLSldhaAhCOOeffz4QBM0W1j9BSyC98cYbADz99NPs2bMn3mLGBfWKn3vuuQWOUdvxuHHj0k6TU9R22rFjR1+DU695ODNnzgSC0mDpRCw2uvVAKxGpKF7hrbbAamA+cH5oTA9gWmwiGoZhxEYsNrrFIjIVWAbsBj7C09DeBF4SkTtD256Mh6BGbi688MJUixA31F4XbrfLy5dffgng27qee+65xAtWTNSjrAUm1bYVrqFq8Kxqe5kQK6eVgZ1zhfa2UI9yXrTf8Hnnned/j8lefcQUMOycuwPIawH/Ejguls9NBFrtVd3hDRo04KCDDgLgm2++SZlcexs33HADECy3wyuvaFiJ5rWuX7+eYcOGAUEl33SphpyXY445xm+OE6mtnwZzq8No6tSp+cakK6+88goQOWf1119/BbwKNGpO0GWt1kvU0Jpbb72VqlWrJlzeSFiuq2EYWY9E22YtoUIkwRlx+OGHA0E13dq1a/thCTfffDMQ1G7LBHJycjjggAOAIPj24osvTqVIexWaX6wpXY888gj77LNPxLEfffQRQ4YMAWDWrFnJETAB9OvXz18FrVy5Eggq/9x///1+ZeGvv/4aCJas2pDq0UcfZcCAAfEWa6lz7tiiBplGZxhG1rPXJPWvXbsWgHPOOQeARYsW+cHDeufR14ZRFNqprLBA4HvuuQeA8ePHZ0Rl4KLQnhaR2LZtmx/ypPXo1H6nIUX/+c9/EitgIZhGZxhG1rPXaHSKphmdcMIJdOvWLdc+TabOW8U23dHSTUZqUY/w/fffDwTXU6bUk4uF//73v7Rt2zbVYhTIXjfR6cX44YcfZkQmQUGsWLGCNWvWAKR9rmc2orXz1IFVvnx5P3RmwoQJqRLLKABbuhqGkfXsNeElhmFkJRZeYhiGATbRGYaxF2ATnWEYWY9NdIZhZD020RmGkfXYRGcYRtZjE51hGFmPTXSGYWQ9NtEZhpH12ERnGEbWs9cl9RuZzQsvvABAy5YtAa+q8pIl1hrYKBzT6AzDyHqySqOrX78+4FV2bdy4MQCnnnoqEFQ7NTIL7c2gfT0uuugiAK699loAPv7449QIZmQUptEZhpH1ZJVGpzX6u3Xr5vfNNE0ucxERv9rzwIEDATjxxBMB+OCDDwDYs2dPaoQzMoqsmOi0laF2r3fO+c1wjMylUaNGfgPrsWPHArB48WIA/vrrr5TJZWQetnQ1DCPryQqN7rbbbsv1euLEidx5550pkiZ1/O1vfwPgnXfeAbwGQMr3338PwB133AGkd1+DI444AoC3336b//3vf4DXPBlMk0tnSpcuDcCVV14JwO233w7Azz//DEDbtm358ccfi/ycChUqALBz5864yVakRiciT4nIJhH5JGxbdRGZLSLrQs/VQttFRB4Wkc9FZIWINI+bpIZhGCWkyJ4RInIK8Bvwb+dc09C2e4EtzrlRIjIQqOacGyAinYDrgU7A8cBDzrnjixSihD0jjj76aADeffddAKpUqQLABRdc4DsjspmKFSsCsO+++wJBK8fff/8dgOXLl3PssV45/apVqwKwZcsWwGsy/MUXXyRV3qIoW7YsALNnzwa8cKHTTjsNSL+WjmoX7tu3LwClSpXyu4DtjZQpU4bBgwcDgSaXl08++cS3sRbGUUcdBcB5550HwHfffVfY8Pj0jHDOLQS25Nl8NvBs6O9nga5h2//tPN4HqopInaKOYRiGkUhKaqOr5ZzbCOCc2ygiNUPb6wLfho3LCW3bmPcDRKQ30LuExwegYcOGAFSqVAnAX/+///77Jfo8vUufcMIJdOrUCYB7770XgKVLl8Yiatxp3769fwfVkIuTTjoJgEWLFuUbP3nyZCC4S1511VUMGjQoGaIWiWpy7733HgDVq1cHoF27dmmnySnNmjUD4LrrrgPgp59+4tFHHwXYqzz+devWBaBXr175bOV5adq0KU2bNo36s88//3wAHnrooZILGCLezgiJsC3istQ5NwGYAPFrd/j0008Dxe+MrhPEq6++CgQ/NIAuXboAcOaZZwIwb968mOUsCepo6NChA+A5E3TJ+sknnvm0sCyB1157DQgmuiOPPDJhshaH8uXL+w24mzf3TLr6P3755Zcpk6u41KhRg6uvvhqAVq1aAV6YU0GISJFj+vfv7/+t33G6xIUeeOCBQOD4UqUjHujv98knn4zbZ5Y0vOQHXZKGnjeFtucA9cPG1QM2lFw8wzCM2CmpRjcd6AGMCj1PC9t+nYi8hOeM2KZL3ERwzDHH5Hpd0pCSN998EwiWwGvXruWAAw4AoHLlygD861//AlKn0en/+vLLL/vb1Gh/9tlnA/DHH38U+H418CrffvttASOTS82aNenRowcQOJXmzp2bSpFKjDojVFvbvn074C1r81KqlKdjFBYuM2PGDMBzNqnWqzm+qaJBgwZA8JuJpMktXLgQCFYRxx13XNSfv27dOl+T++2332IRNRdFTnQi8iLQGthPRHKAO/AmuMki0hNYD1wQGv4Wnsf1c2AHcEXcJDUMwyghRU50zrmLC9jVNsJYByTtltO6dWsguIMW136hlU1Ua1ONaMyYMX6oxocffgjgG1EPOuggAL755psYJC8+3bt3z/X6+++/9934BWly5cuX9w26PXv2BILgTTWcp5rBgwf7dirV7IpLtWrVgCDcpohwhLiwfPlyILgO9LqAQDO97777AHjrrbdKdAwNkh49enQ+jTwVHHjggb4mp9WBwtFr6/rrrwcCu2I6YClghmFkPRmdAqaaQFFBzwWhnlR9//DhwwF44okn8h2jdu3aANSp44UFJluj01p7ypVXXulX8MiLelRPOOEExo0bl2ufhnCsWrUqAVJGT9u23oKgZ8+evpetJDaZNm3a+PYr1eg2bPD8Xxp+o57BeKIhJKpp63mFwP6mtqqSoisOEfFXLalA/5+rrroqoianaFC6hjdpuuGYMWMSLGHRmEZnGEbWk9EaXaxoWSdNHp4/f34qxSkWEyZMKNA2p1V5Nc4unShXrhyQu6TWFVd4PqtI3sm8aHC09o4oW7Yszz//PBB4kjWp/IwzzgASo9EpGkg+Y8YMf4Vw8sknA8H5L6n38KyzzgK8c6SrjVSgaYSqIReEap2qWWstwddffx1IbVxkRk50LVq0APKHlxSHzp07+8tRDUspaUZFMnj44YcBLyMCoF69evnGjB8/HgguuD59+vj7NGvklltuSaicRaEhGFdddRXgLTMLCyepUaNGrvcNGTIECKqv3H333flCZXTJN23aNBLNn3/+CcCnn37qT3SKGuW1aGi03Hjjjblez5s3z6/ikgpUEdixY4c/iUXDIYccAgTfmd6AUoEtXQ3DyHoyUqPbuNGLQf7hhx+AIB1Fw0SiCTOpUKECZcp4/75qiOHoZ6l2pM6HZDshFA0OVi22W7durFixAgiCN3fs2AEES7ZwjW7KlCkAKW8NqM4cRTXVSFStWpWZM2cCXrVhCAJm1WG0e/duf7wu2TVtr7iaVCwMGzbMXyFoALem7UWLaq+aSqb89NNPcQ2eLS4rV64EvOtKZdRlqX4Pu3bt4qabbgLwK86kE6bRGYaR9WSkRqfhA2oI1tALrZ6g6VpFoaEjkWx9akPSMRMnTgQCbTLZaBMY1eL0ORLq1i9VqpSfYpSq1LWieOqppwrcd9lll3HooYcC0LWrVwmssP9DK4ls3boVCFKwksGOHTv8gGe1EWrgcLRoK8e8aVXTp0+Pg4SxE14VR9PTlBo1auQKy0o3TKMzDCPryUiNTtEEd73bq5dr5syZMWswee+q69ati+nzksHo0aOBoFfE7t27/YqummCdbmh/gIL2aRpRYd+nfv/a5FqDkVOlfRdXk1NOOeUUILALxyvwOBmULVuW/fffP9c29YY/8MADqRApF6bRGYaR9WS0RqfeRu1wpR69tm3bFkuj0/dpHJSI+EnwmYCm3qgnTO2KmzZt8ouKpgt5E+4vv/xyRowYEXFs7dq1fXtspH3g2VK7desGBKlGWoghU9Hv79NPPwXi2w0r3mjnr0hFMrUQrHptU0lGT3QaTqHufL3ABw4c6Od7nnvuuUDuMATwqk/oj0iDb1u2bAl4ZaE19ETDMdKx2Y66+jXyXJeB6oBQB0o68dxzzwFBU5levXr5oS/6w1bWrVvnO4VGjhwJBE4ZvRGVLVvWv0FF03glXalcuXKuytYQ3Mi1oVE6oiE8WhUagutP2xCkA7Z0NQwj68lojU7RMBPVYHr27Ok3t9E+EJrvuGDBAsAzVOs+dWJoqkr4Zx5/fJHdGlOGhlP84x//yLVde2cMHTo02SIViaai9erVC4BZs2YxZ84cIEgR0uDYPXv2+GaJAQMG5PqczZs3A17NtkzW5JSmTZvmC7RV7TfZqClEa+ypcyS8J4k2xQnX5BQ1RaQybS0vptEZhpH9OOdS/sDrFBbzo3z58q58+fJu+PDhbtOmTW7Tpk1u9+7dbvfu3W7Pnj1uz549buvWrW7r1q1uzZo1+fbp63feeceVK1fOlStXLi5yJeJRp04dl5OT43Jycnz5165d69auXevq16/v6tevn3IZo3mMHTvWKX/99VeRj/Hjx7vx48e7Vq1auVatWqVc/ng9Ro4c6V9/+kiVLKtWrXKrVq3yr6uFCxe6hQsXuiZNmrhGjRq5Ro0auQ0bNrgNGzb4Y/bs2eOWLFnilixZ4urWrevq1q2bLHmXRDPHmEZnGEbWIyWtzhtXIeLU1zWcmjW9ntraY1M9c5HCRjRVSGt+PfPMM1HVRkslb7/9tp+8r2gKktZnM9IfbZo+c+ZM3yamAcdt2rRJiUw6J+TtUDZ27Fj/GstbsGD58uV07twZoMCQoASx1Dl3bFGDsnaiyzY0bkwdELfccotfxHL16tVAUCCxsLaHRnpxwQVeA70XX3zR36aNqx988MGUyKRhPtE0pdaS/O3atfOrCSWZqCY6W7oahpH1ZEV4yd7AJZdcAsCgQYP8barJ6RLWNLnMRr+/VOXoKhqAfuuttxY4RjVQrVidIm0uakyjMwwj6zGNLkMID9YEz46idedSrQEYJUeDc8HLTYagKk+q0KrPkTS6NWvWAPlr/6U7ptEZhpH1FKnRichTQGdgk3OuaWjbaOAs4E/gC+AK59zW0L5BQE9gD3CDc25WgmTfq9BqLFotwsgOtPgEpE8lYQ0PyaZrLRqN7hkgb0LbbKCpc+4oYC0wCEBEmgAXAUeE3vOYiGTP2TIMIzOJMkWrAfBJAfvOASaF/h4EDArbNws4IVkpYPawhz32ukfSUsCuBN4O/V0XCO8mnBPaZhiGkTJi8rqKyGBgNzBJN0UY5gp4b2+gdyzHNwzDiIYST3Qi0gPPSdHWBXlkOUD9sGH1gIiJb865CcCE0GdFnAwNwzDiQYmWriLSARgAdHHO7QjbNR24SETKi8jBQEPgg9jFNAzDKDnRhJe8CLQG9hORHOAOPKdDeWB2qPro+865q51zq0RkMrAab0l7rXNuTxRybAa2h57Thf1IH3nSSRYweQojnWSB7JfnoGgGpUX1EgARWRJNFYJkkU7ypJMsYPIURjrJAiaPYpkRhmFkPTbRGYaR9aTTRDch1QLkIZ3kSSdZwOQpjHSSBUweII1sdIZhGIkinTQ6wzCMhJAWE52IdBCRz0TkcxEZmORj1xeR+SKyRkRWiciNoe3VRWS2iKwLPVdLslylReQjEZkRen2wiCwOyfOyiJRLkhxVRWSqiHwaOkcnpPLciEi/0Pf0iYi8KCL7JPPciMhTIrJJRD4J2xbxfIjHw6HreoWINE+SPKND39cKEXlNRKqG7RsUkuczEWmfaFnC9vUXESci+4VeJ/zchJPyiS5U3WQs0BFoAlwcqoKSLHYDtzjn/g60Aq4NHX8gMNc51xCYG3qdTG4E1oS9vgd4ICTPz3ilsJLBQ8BM51xjoFlIppScGxGpC9wAHBsqGVYar1pOMs/NM+Sv5lPQ+eiIFzTfEC/dcVyS5ElVdaFIsiAi9YF2wPqwzck4NwFp0Lz6BGBW2OtcFVBSIM+00JfyGVAntK0O8FkSZaiH94NpA8zAyyHeDJSJdM4SKEcV4CtCttyw7Sk5NwRFI6rjBbvPANon+9yQp5pPQecDeBy4ONK4RMqTZ1/M1YVilQWYineT/BrYL5nnRh8p1+hIo4onItIAOAZYDNRyzm0ECD3XTKIoDwL/ArSxZg1gq3Nud+h1ss7RIcCPwNOhZfQTIlKJFJ0b59x3wH14msFGYBuwlNScm3AKOh/pcG2ntLqQiHQBvnPOLc+zK6mypMNEF3XFk4QKIfI34BXgJufcL8k+fpgcWs15afjmCEOTcY7KAM2Bcc65Y/DS9JK9hPcJ2b7OBg4GDgAq4S2B8pIuoQQpvbZjqS4Up+NXBAYDt0fanUxZ0mGii7riSaIQkbJ4k9wk59yroc0/iEid0P46wKYkiXMi0EVEvgZewlu+PghUFRHNTU7WOcoBcpxzi0Ovp+JNfKk6N6cDXznnfnTO7QJeBf5Bas5NOAWdj5Rd2xJUF7rUhdaGKZDnULyb0vLQ9VwPWCYitZMtSzpMdB8CDUOes3J4xtKkFc8XEQGeBNY458aE7ZoO9Aj93QPPdpdwnHODnHP1nHMN8M7FPOfcpcB84PxkyuOc+x74VkQahTa1xSvYkJJzg7dkbSUiFUPfm8qT9HOTh4LOx3TgspCHsRWwTZe4iUTSpLqQc26lc66mc65B6HrOAZqHrqvknptEGf+KacDshOcd+gIYnORjn4SnMq8APg49OuHZxeYC60LP1VNwXloDM0J/H4J3UX4OTAHKJ0mGo4ElofPzOlAtlecGGAZ8CnwCPIdXRSdp5wZ4Ec8+uAvvh9uzoPOBtzwbG7quV+J5i5Mhz+d49i+9nseHjR8ckuczoGOiZcmz/2sCZ0TCz034wzIjDMPIetJh6WoYhpFQbKIzDCPrsYnOMIysxyY6wzCyHpvoDMPIemyiMwwj67GJzjCMrMcmOsMwsp7/B+sMPMxYZHdWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showimages = iter(train_loader).next()[0][:20]\n",
    "grid_img = torchvision.utils.make_grid(showimages, nrow=5)\n",
    "plt.imshow(grid_img.permute(1, 2, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a is a <class 'list'>\n",
      "torch.Size([100, 1, 28, 28])\n"
     ]
    }
   ],
   "source": [
    "a = iter(train_loader).next()\n",
    "print(\"a is a {}\".format(type(a)))\n",
    "print(a[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train]\n",
      " - Numpy Shape: (60000, 28, 28)\n",
      " - Tensor Shape: torch.Size([60000, 28, 28])\n",
      " - min: tensor(0.)\n",
      " - max: tensor(1.)\n",
      " - mean: tensor(0.1307)\n",
      " - std: tensor(0.3081)\n",
      " - var: tensor(0.0949)\n"
     ]
    }
   ],
   "source": [
    "train_data = train_dataset.data\n",
    "train_data = train_dataset.transform(train_data.numpy())\n",
    "\n",
    "print('[Train]')\n",
    "print(' - Numpy Shape:', train_dataset.data.cpu().numpy().shape)\n",
    "print(' - Tensor Shape:', train_dataset.data.size())\n",
    "print(' - min:', torch.min(train_data))\n",
    "print(' - max:', torch.max(train_data))\n",
    "print(' - mean:', torch.mean(train_data))\n",
    "print(' - std:', torch.std(train_data))\n",
    "print(' - var:', torch.var(train_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feedforward Neural Network Model Structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_classes):\n",
    "        super(Net, self).__init__()                    # Inherited from the parent class nn.Module\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)  # 1st Full-Connected Layer: 784 (input data) -> 500 (hidden node)\n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size) # 2nd Full-Connected Layer: 500 (hidden node) -> 10 (output class)\n",
    "        self.fc3 = nn.Linear(hidden_size, num_classes)\n",
    "        self.af = nn.ReLU()                          # Non-Linear ReLU Layer: max(0,x)\n",
    "    \n",
    "    def forward(self, x):                              # Forward pass: stacking each layer together\n",
    "        out = self.fc1(x)\n",
    "        out = self.af(out)\n",
    "\n",
    "        out = self.fc2(out) #1\n",
    "        out = self.af(out)\n",
    "        out = self.fc2(out) #2\n",
    "        out = self.af(out)\n",
    "        out = self.fc2(out) #3\n",
    "        out = self.af(out)\n",
    "        out = self.fc2(out) #4\n",
    "        out = self.af(out)\n",
    "        out = self.fc2(out) #5\n",
    "        out = self.af(out)\n",
    "        out = self.fc2(out) #6\n",
    "        out = self.af(out)\n",
    "        \n",
    "        out = self.fc3(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 不用class而直接建構網路"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nnet = torch.nn.Sequential(\\n    torch.nn.Linear(input_size, hidden_size),\\n    torch.nn.ReLU(),\\n    torch.nn.Linear(hidden_size, hidden_size),\\n    torch.nn.ReLU(),\\n    torch.nn.Linear(hidden_size, num_classes)\\n)\\n'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# net = Net(input_size, hidden_size, num_classes)\n",
    "'''\n",
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(input_size, hidden_size),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(hidden_size, hidden_size),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(hidden_size, num_classes)\n",
    ")\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net(input_size, hidden_size, num_classes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose the Loss Function and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the FNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/10], Step [100/600], Loss: 0.6329\n",
      "Epoch [1/10], Step [200/600], Loss: 0.3176\n",
      "Epoch [1/10], Step [300/600], Loss: 0.2060\n",
      "Epoch [1/10], Step [400/600], Loss: 0.2437\n",
      "Epoch [1/10], Step [500/600], Loss: 0.2443\n",
      "Epoch [1/10], Step [600/600], Loss: 0.0652\n",
      "Epoch [2/10], Step [100/600], Loss: 0.2070\n",
      "Epoch [2/10], Step [200/600], Loss: 0.1252\n",
      "Epoch [2/10], Step [300/600], Loss: 0.0581\n",
      "Epoch [2/10], Step [400/600], Loss: 0.0341\n",
      "Epoch [2/10], Step [500/600], Loss: 0.0637\n",
      "Epoch [2/10], Step [600/600], Loss: 0.0626\n",
      "Epoch [3/10], Step [100/600], Loss: 0.0656\n",
      "Epoch [3/10], Step [200/600], Loss: 0.0658\n",
      "Epoch [3/10], Step [300/600], Loss: 0.0737\n",
      "Epoch [3/10], Step [400/600], Loss: 0.0383\n",
      "Epoch [3/10], Step [500/600], Loss: 0.0131\n",
      "Epoch [3/10], Step [600/600], Loss: 0.1148\n",
      "Epoch [4/10], Step [100/600], Loss: 0.0249\n",
      "Epoch [4/10], Step [200/600], Loss: 0.2689\n",
      "Epoch [4/10], Step [300/600], Loss: 0.1194\n",
      "Epoch [4/10], Step [400/600], Loss: 0.0473\n",
      "Epoch [4/10], Step [500/600], Loss: 0.0876\n",
      "Epoch [4/10], Step [600/600], Loss: 0.0811\n",
      "Epoch [5/10], Step [100/600], Loss: 0.0208\n",
      "Epoch [5/10], Step [200/600], Loss: 0.0031\n",
      "Epoch [5/10], Step [300/600], Loss: 0.0023\n",
      "Epoch [5/10], Step [400/600], Loss: 0.1324\n",
      "Epoch [5/10], Step [500/600], Loss: 0.0255\n",
      "Epoch [5/10], Step [600/600], Loss: 0.0836\n",
      "Epoch [6/10], Step [100/600], Loss: 0.0698\n",
      "Epoch [6/10], Step [200/600], Loss: 0.0200\n",
      "Epoch [6/10], Step [300/600], Loss: 0.0348\n",
      "Epoch [6/10], Step [400/600], Loss: 0.0401\n",
      "Epoch [6/10], Step [500/600], Loss: 0.0271\n",
      "Epoch [6/10], Step [600/600], Loss: 0.0615\n",
      "Epoch [7/10], Step [100/600], Loss: 0.0297\n",
      "Epoch [7/10], Step [200/600], Loss: 0.0146\n",
      "Epoch [7/10], Step [300/600], Loss: 0.0248\n",
      "Epoch [7/10], Step [400/600], Loss: 0.0282\n",
      "Epoch [7/10], Step [500/600], Loss: 0.1411\n",
      "Epoch [7/10], Step [600/600], Loss: 0.0444\n",
      "Epoch [8/10], Step [100/600], Loss: 0.1055\n",
      "Epoch [8/10], Step [200/600], Loss: 0.1068\n",
      "Epoch [8/10], Step [300/600], Loss: 0.0040\n",
      "Epoch [8/10], Step [400/600], Loss: 0.0030\n",
      "Epoch [8/10], Step [500/600], Loss: 0.0564\n",
      "Epoch [8/10], Step [600/600], Loss: 0.0712\n",
      "Epoch [9/10], Step [100/600], Loss: 0.0040\n",
      "Epoch [9/10], Step [200/600], Loss: 0.0066\n",
      "Epoch [9/10], Step [300/600], Loss: 0.0326\n",
      "Epoch [9/10], Step [400/600], Loss: 0.0178\n",
      "Epoch [9/10], Step [500/600], Loss: 0.0154\n",
      "Epoch [9/10], Step [600/600], Loss: 0.0029\n",
      "Epoch [10/10], Step [100/600], Loss: 0.0051\n",
      "Epoch [10/10], Step [200/600], Loss: 0.0091\n",
      "Epoch [10/10], Step [300/600], Loss: 0.0158\n",
      "Epoch [10/10], Step [400/600], Loss: 0.0125\n",
      "Epoch [10/10], Step [500/600], Loss: 0.0011\n",
      "Epoch [10/10], Step [600/600], Loss: 0.0395\n",
      "CPU times: user 8min 11s, sys: 1min 2s, total: 9min 13s\n",
      "Wall time: 2min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch.nn.functional as F\n",
    "losses = []\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):   # Load a batch of images with its (index, data, class)\n",
    "        images = torch.FloatTensor(images.view(-1, 28*28))         # Convert torch tensor to Variable: change image from a vector of size 784 to a matrix of 28 x 28\n",
    "        labels = torch.LongTensor(labels)\n",
    "       \n",
    "      \n",
    "        optimizer.zero_grad()                             # Intialize the hidden weight to all zeros\n",
    "        outputs = net(images)                             # Forward pass: compute the output class given a image\n",
    "        loss = criterion(outputs, labels)                 # Compute the loss: difference between the output class and the pre-given label\n",
    "        losses.append(loss.item())\n",
    "        loss.backward()                                   # Backward pass: compute the weight\n",
    "        optimizer.step()                                  # Optimizer: update the weights of hidden nodes\n",
    "        \n",
    "        if (i+1) % 100 == 0:                              # Logging\n",
    "            print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'\n",
    "                 %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7febe10f9a90>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXJwkgyCYSBQENKkK1bkBZWrdaV/TW/mpt1Var1sttrbW2995eqNX2qq2tdvFa930trlSpgCiIAiJL2HcJe4SQhCUkhJCQfH9/zJnJJJlJJskkM+f4fj4eeWTmzHdmvl+YvOd7vt/vOcecc4iISLBkpLoCIiKSfAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAJI4S4iEkBZqXrj3r17u5ycnFS9vYiILy1atKjYOZfdVLmUhXtOTg65ubmpensREV8ysy2JlNOwjIhIACncRUQCSOEuIhJACncRkQBSuIuIBJDCXUQkgBTuIiIB5LtwLzlQxfiJy9lXUZXqqoiIpC3fhfudb69kwoJtDL9neqqrIiKStnwX7iOP7wVAZXVNimsiIpK+fBfuVw0bAMCIgb1SXBMRkfTlu3DvmJVBl46ZnNavR6qrIiKStnwX7gCZZlQ7l+pqiIikLV+Ge0aGUVOjcBcRiceX4Z6ZoZ67iEhjfBnuGWZosYyISHy+DPfMDDQsIyLSCH+GuyZURUQa5ctwNzNqFO4iInH5MtwzMgBlu4hIXL4Md0M9dxGRxvgy3DNMHXcRkcb4NNwNLZYREYnPl+GOoWEZEZFG+DLcM0zjMiIijfFpuKvnLiLSGF+Gu1bLiIg0zp/hbqBsFxGJz6fhrtUyIiKN8WW4ZxhoRlVEJL6sVFegJVZt38eq7ftSXQ0RkbTVZM/dzAaY2UwzW2Nmq8zs5zHKmJk9ZGZ5ZrbczIa2TXVFRCQRifTcDwH/6ZxbbGbdgEVm9oFzbnVUmUuBQd7PSOAx77eIiKRAkz1359wO59xi73YpsAboV6/YFcCLLmQe0NPM+ia9tiIikpBmTaiaWQ5wJjC/3kP9gG1R9/Np+AWQdNv3HmjrtxAR8aWEw93MugJvAbc75+rPZlqMpzRYzmJmY80s18xyi4qKmlfTGA5UVbf6NUREgiihcDezDoSC/RXn3MQYRfKBAVH3+wPb6xdyzj3pnBvunBuenZ3dkvrWcXhHXy72ERFpc4msljHgGWCNc+6vcYpNAq73Vs2MAkqcczuSWM+YdB1VEZHYEun6fg24DlhhZku9bb8GjgVwzj0OTAHGAHlAOXBj8qvaUI0OUxURianJcHfOzSH2mHp0GQf8NFmVSpROHiYiEpsvTz8QVq2eu4hITL4Od/XcRURi83W4V9ekugYiIunJ1+F+z7urmy4kIvIF5Otwn5NXnOoqiIikJV+Hu4iIxKZwFxEJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJAvgz3t34yOtVVEBFJa74M92HH9Up1FURE0povw11ERBqncBcRCSCFu4hIACncRUQCSOEuIhJACncRkQBSuIuIBJBvw/3SL/dJdRVERNKWb8N96soCAJxzKa6JiEj68W24h+UVlqW6CiIiacf34d4pKzPVVRARSTu+D3ezVNdARCT9+D7cq2s05i4iUp9vw/1PV54KQLUmVEVEGmgy3M3sWTMrNLOVcR4/z8xKzGyp93NX8qvZUGV1KNSf/2Rze7ydiIivJNJzfx64pIkys51zZ3g/d7e+Wk0rKDkAwIQFW9vj7UREfKXJcHfOzQJ2t0NdmiUzI1T1QxpzFxFpIFlj7qPNbJmZTTWzU5L0mo3KytAyGRGReLKS8BqLgeOcc2VmNgZ4GxgUq6CZjQXGAhx77LGtetNMhbuISFyt7rk75/Y558q821OADmbWO07ZJ51zw51zw7Ozs1v1vgp3EZH4Wh3uZtbHLHQokZmN8F5zV2tftykalhERia/JYRkzmwCcB/Q2s3zgt0AHAOfc48B3gJ+Y2SHgAHC1a4ezeSncRUTiazLcnXPXNPH4w8DDSatRgjIzfXv8lYhIm/NtQqrnLiISn8JdRCSAfBvuHTQsIyISl28TUuEuIhKfbxMyK1PDMiIi8fg23M89KXQQ1GWn9k1xTURE0o9vw/2wDqHL601esSPFNRERST++DXcREYlP4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQmgrFRXoDVGDuyV6iqIiKQl3/fcXaorICKShnwd7hlmSncRkRh8He5mUOOU7iIi9fk63Kuqayg5UJXqaoiIpB1fT6gu3Lwn1VUQEUlLvu65i4hIbIEI98pDNamugohIWglEuJ/0m6kaexcRiRKIcAfYVXYw1VUQEUkbgQl3LYgUEakVmHAXEZFaTYa7mT1rZoVmtjLO42ZmD5lZnpktN7Ohya9m0yoP1XCgsjoVby0iknYS6bk/D1zSyOOXAoO8n7HAY62vVvONeWg2X7rrvVS8tYhI2mky3J1zs4DdjRS5AnjRhcwDeppZ32RVMFE6C4GISK1kjLn3A7ZF3c/3tjVgZmPNLNfMcouKipLw1iIiEksywt1ibIvZj3bOPemcG+6cG56dnZ2EtxYRkViSEe75wICo+/2B7Ul4XRERaaFkhPsk4Hpv1cwooMQ5tyMJrysiIi3U5FkhzWwCcB7Q28zygd8CHQCcc48DU4AxQB5QDtzYVpUVEZHENBnuzrlrmnjcAT9NWo1ERKTVdISqiEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgHk63C/5bwTUl0FEZG05OtwH9CrS6qrICKSlnwd7iIiEpvCXUQkgHwd7rFOJC8iIj4PdxERic3X4W7quouIxOTrcBcRkdgU7iIiAeTrcDdNqYqIxOTrcBcRkdh8He6nD+iZ6iqIiKQlX4f74D7dUl0FEZG05OtwFxGR2BTuIiIBpHAXEQmgwIZ7dY3j/VUFOOdSXRURkXYX2HB/evZGxr60iCkrClJdFRGRdhfYcN++9wAARaUVKa6JiEj7C1y4Dxw/mTnri9FgjIh8kQUu3J2DRz/KS3U1RERSyvfhrtP+iog05Ptw79P9sEYfN6W/iHwB+T7cXx07qtHHfztpFXmFZe1UGxGR9OD7cO/bo3PM7dHL2y/468ds213eTjUSEUk934d7ZkZiwy4bi/e3cU1ERNJHQuFuZpeY2TozyzOzcTEev8HMisxsqfdzc/KrGluscJ+7YRcvzdvSXlUQEUk7WU0VMLNM4BHgQiAfWGhmk5xzq+sVfc05d2sb1FFERJopkZ77CCDPObfROVcJvApc0bbVSj6dY0ZEvkgSCfd+wLao+/netvquNLPlZvammQ1ISu1ERKRFEgn3WDOW9bvB/wJynHOnAdOBF2K+kNlYM8s1s9yioqLm1TQJnHP8fcZ6tu7SyhkRCbZEwj0fiO6J9we2Rxdwzu1yzh307j4FDIv1Qs65J51zw51zw7Ozs1tS31bZUVLBXz74jBueX9Du7y0i0p4SCfeFwCAzG2hmHYGrgUnRBcysb9TdbwJrklfF5KmuCe1wHKyqSXFNRETaVpOrZZxzh8zsVmAakAk865xbZWZ3A7nOuUnAbWb2TeAQsBu4oQ3rLCIiTWgy3AGcc1OAKfW23RV1ezwwPrlVa1tjX8xlX0UVr44dneqqiIgkne+PUAUYffyRTZbZWLSf+6eti9x/f/VO5m3czTNzNrVl1UREUsJStf57+PDhLjc3NymvtWjLHq58bG6Ln3/uSdm8cNOIpNRFRKQtmdki59zwpsoFouc+7LgjWvX8jz9r/2WZIiJtKRDhLiIidSncRUQCSOHu2Vy8n7cW5ae6GiIiSZHQUsgvgvP+/BEAVw7rn9qKtIFJy7Yz+OhuDO7TLdVVEZF2op57Kzjn2L2/stWvkzNuMn+Y0nYH9d42YQkXPzirzV5fRNKPwr0Ju8oOxn3sjdx8ht7zAau372vw2Kbi/QmdZjh8+b8nZ21seSVFROpRuNezbXc5by7Kp+zgIeasL2bYvdP5cO3OmGVn5xUDsL6wFICS8iogtO7+63/+iJcbuRrUS59uZm3BPs6+f2ZyGyAigsbcGwiH7WMf5fGlvt2BUFifP+ToRp+3ansJlz00h79cdXrkfMhLtu7lujhnN7jznVXJqrIk0Wc7SzmiS0eyu3VKdVVEWkU99zg2FO3n3eU7IvcrqqoblIkedllXEOq9z15fFDkBvq795D8X/W0WZ9//YaqrIdJqgQn3Zb+9qM1ee8KCbQy5873I+DjAk7M2RMJ/3sZdWNQlTSzG5U3mbijmwemfJa1OVdU1/PL1pWwu3l9nu3OOe95dHXMeQBJToVNCSwAEZlimR+cObfba4RUxG4rKuOapeWRlGJujruY0YcE2+h/RBYDissrIeeOje/bXPjUfgNsvOCkpdVq0ZQ8TF39O/p4DvP4ftWM/e8qreGbOJt5anM/Suy7inaWfN9qurbvLOWNAz6TUqa2VHKhq0//ndFJaUcWK/BK+emLvVFdFfCowPff2cMNzC8nfc6BOsIc94J1xck5eMf/95vL2rlpE+AslvPPw81eX1nm8oqo68mV15WNz+dYjn7BnfyWvLtja4vf8JK+YnHGTeSN3W9OFW+hfy7Zz+v++z4r8kjZ7j3Ry24QlXPv0fIpK46/WkuQqrajiUHVw9toU7m0olWPue8qrGH3fjAbbr31qHkPv+QAILdcE+MXrSxk3cUVk3gBCveTLHppNXmFZk+/1/adDeyVt+aU2e33o5G6rd3wxwv2znaF/9/BczzcfnsOzOj11mzr1d+/zq7dS1zFLNoV7G2ru2ZRjnZ3y3Adm8qf31ib82tHr5XeUVDR4fPHWvQ22FXtr+SsP1fZaZq4tZNX2ffz9w/VN1rs9JesM1au376O88lCLnz93Q2hvpSDGv3FrVR6q4fO9B+psW55fwt3vrk76e0ldExfHH8b0m0CFe/fD0msKYVPxforLDrK3vPYo1reXfM7lf58ds/wPn6174e5V20vYsqucxz7akPB7PtGKg6FcO+9rTFtVwIJNuyP3a2pcnUnrsMnLd/B6bvLO+1NRVc2Yh2bzk5cXt/g1wscw5G7ZzX1T1/Deyh1NPCNx0Z8XaR+puq5FWwpUuB/Ts3Oqq1DHis9LGH7vdM64+4PItttfW8rKz+OvZFm0ZTc54ybzztLPueyhOXHLhVfkLNi0O+EeaPSYeKylnW2ltKKqToiH/cdLi/juE59G7t/4/ELOvn8mL326uU65n/6j5SEcS6U3rrpoy54Wv4Z5sxrOwRMfb+THrfiiaEwAM6fNzV5fxO8mNe84kuh/5/DBiM1RU5N+/1GBCveaAPwlvPhpqEdYfyI0bPHWPdw3dU2dc9E8NCOP9TtLY5aPFj0m/u8v1l4FK/xlc9PzC8krLE16L+aWVxbz3Sc+jfzR1NS4mOfSCQ9L/XNJ2+4ah5sXa8lqwhJ87rkPzOTJWYnveUnrXffMAp6fu7nFz88ranqeKdqSrXs4/tdTmOsdsZ4uAhXuXTul17BMSyyPsxpk3sZdAHz70bk88fHGOuUe/3gDF/6teScGm72+4QexuKySC/46i1++vozSg03vDawt2EdpRdO9nDU7Ql8e4R7zup2ljZ5Lx5qZuou37uGJjxMP0PorihJRUVVN4b6G4+tNfQ1u2VXOH6Y0nDNJlMNFltYm6pX5W/iBN8kNoSWvOeMm89NXFpMzbnLMoa+wqSt2MOoPM6gK0KqRRLSmO/Op97f58fqGc2ZzNxRH5rTaW6DC/bEfDOPXY4akuhqtsqneQUlhL8zdXGfCM5ZEgjYR/1zyOXe+vbLOtmfnbOKRmXmR+yXlVVzy4GxO/d37jb7Wy/O2UFwWGkMOj+nX/7Bv211OzrjJkfvh0G3spG3Rvv3oXO6bGgrQVxdsbfK8/LU998Tj/YbnFjDiD7WrjyJHIbfF3mJUtZyDh2Y0b1L7jn+uZE5UL3LLrtBnavKK0LzA0m0NJ9XD7nxnJQX7KtjbgqGJWHaUHODgofYbAmypthpzv/ap+XwvauixPQUq3I/ufhjXj85JdTXaxNSVBYz8w/RGy9zyStuM+wLc/e7qyFp+gNKDif3x/ybqSyJ38x6enr2R656pO3E8P8Z4/FuL8hl27/QG69qb+hMcN3EF//nGskbLhF8jo162N3ZU77yNdevY3L2LlvrHgq2saqejjR+asb7BF3FrVB6qYfR9H/Jfb8ReXphXWErOuMmRvdKg2lBUt8P2/qqCdjl+IVDhDpAV9Rf71ROOTGFNkm9PE72pWEMtrTV99c464/mvLdza6GqO0ooq9uyvZG1Bw0C65ZXF3Du54Vh7rMndT7ye57oE5hJieWfp5+SMm8w3/vIR1zw5r07vcdqqAiAU0GsL9jHF69GOeajuKqZtu8v5dENqgyc0fNWyoI2357Nk6152lR1kY1EZ1z41L/Lv8dcPknd6DIBDNaE9zemrY59Vda73b/vu8u1Jfd+WaM1XWWRPsN5AX6y9gYqqasa+tIjrnpnf4LFkC164Z9Y26R//PiqFNQmG/ZXVdcbz/+etFXVW/9R3+d/ncPnf53DJg7GXe8bym3pDQGYw0ZtU3Xcg8eGB6AunvORNTG8o2s+nG3fxR2/YZu6GYsZPXBEpf8mDs7nFG4uu7+z7Z3LNU/MSfv/WKCo9yN3/Ws2ByoZfdLMS/NKurnHMWFMbpMPunU5ZjLmTZz/ZxLB7p3P+Xz5m7oZd/MdLiyIHtoXFCqqHP1zP+IkrOFRdw4r8El5q5JTWiagd2kqs/NqCfc0+P9P4icsZ/JupzatYGwrPn2xtZN4jWfw/AxlD98Oy2FfR8gNUpGnvr4rdG9sSdWqGB6a1bCJx+97aicv6B+5ETy7mFZayMWqXd3bUhFZGvWGTrbvKKa88xH2tmNwEKK8Mnee/0tsTqB9MW3eVk5EBO/dV8MvXa4eHDlRWYwZ/++CzyPmF7vjnCn592Zfo3bUTX/l9aMitR+cOXDNyQJ3XjJ5rCZ8+Itay38c/3lBn6Azgt++s4gejjm2yXU1dUWz2+mL+/H4oWC/40lH86IXQaqvrRh3X5GvH5f0fxcr2d5dvZ9vuA/Tu2pErzuhHx6wMvv3oXMorq/nJeSfQKSszobeYsCD2KTFW5Jcw6OiuHNYh9Dp1/x8T78dvKCqL/JuHP3KLtuymY2YmpxzTvUH59lzPF8hwn/aLcyITk+eclM1nBaUUxFjpIC2XyNGSj8xs2RLA+kdnRvvN2ys5Ibsro084kgv+WneF0PQ1hZHb9YfEZ6wt5OS7prWoPtFuen5hnfH321+rXbJaXHaQcx6IffGVq5+ax6Vf7sMTszbSuWMmvbt2YuKSz+nUIYP7vn1apNyhmhoWb4k/4XnuAzPZue8gm/94WYPHlmxtuG5/b3klr8xv+XmDwsqj9ijKY+xdhE1fvZPC0oNcO7LhF8q/lm1nzY59XD86hz49Dotsr652kT2ncLtu/ceSyOP//eZyNv/xspgreKav3snNL+ay8I4LGpyDP9bqJoDC0gr+7eE5XHHGMfzf1WcCjc8xXP/sAr59Zj++dWa/Bo+9ELXksrrGReoD8N7tZzco35KVWi0VyHDv26MzfXuEejYv3jQCIOZut/jTx58V0S9Gz/Vfy2rHbuv33JOl/sRqtOH3xp/wXrZtL8u8VSoPTo9e/dKwnj9+eVHc19m5LzSO7pxrMKkb/eUWNmNtw23NtWjLnjp1+tmE2uBdv7OU3C17uGZEKMzDwRYr3MPPe9Q74jq8sm1/M08D4Rz8+KVFfG/EgMh69rUF+8jull2nXPTqpnUFpZELxO8/GPpyemfp9ki4R3t05gaeuaFX5P6sz4qY9VkRy/L3cv3oHLodlkXvrp1wzrErao8nr7CszhLfKSsKGtbd+90eE/KBG3OP54HvnNZ0IfGFxz/eELeHHPZpK1dgtN/aZFenR5ro+HN1jWP+xl3kjJvM07M3Rr44kqW47CBD7pzK0m17+ePU+Bdvv+T/ZjN+4go+yStuMLbdWA8favfsoi+K83qcM4vmjJtMVXXoH6fGOd5bVcCNzy2M9LjD8yjxljRe/OCsyPEWsUQ/bcbawpiv89wnm/n6nz9ilPel8cSsjUyOqvvy/Lr/B9FLWN/I3UbOuMmRuSD13JPoquEDuGr4AGpqHMf/ekqqqyNprrFeeDJNWLCtzrjww1HHEjTmxDtqgzTWCqTWuuudlVRU1fCtRz5ptFx4DuT7T9dd/RG9p3ygqjrm4fklMSbLf/Xmcnp16djoe8aajMzfc4ARv59OaSNzbZf+X2iSf9rt5zT6+hDay3h3+Q5+//++3OCxQ15bpq6s2zMPLyONJXx0eGROpB3S/QvTcw/LiFoq+V8XnRQZthGJ54s4pLdwc8vPuxNLczpUN0edGiOW6JVYVYdqvzQKSw9yIIFzJsX6Ag0Hf1h4b+KOf65sUBZCE9vJ3ltKti9cuAMMPbYnPTp34NbzB3HOSXXH6e654pQU1UpEmmvB5vhzIPFEz82ExTsyPJ7WHnh1qLrt1818IcN94i1fi3vN1euijnAddFTXyO2hx/rjUnQikriccZPZGuPKak254bmFrXrfRPYwWusLGe7xvP3TrwFwev8eAPQ7onZFxt+vHZqSOolI25qwsPVLRdNRQuFuZpeY2TozyzOzcTEe72Rmr3mPzzeznGRXtC29cNMIXhs7KnKh6L997wxO79+Dv333DE45pjt/uer0mEvvmvKXq05PqNyInF5NFxKRNtGci+H4SZPhbmaZwCPApcDJwDVmdnK9Yj8C9jjnTgT+Bvwp2RVtS+eelM3I42vPQ3N8dlfeufUsjji8I5NvO5srh/WP+bwhfbrxP5cM4fwhR3H/d07jrBN7M+M/zwUgu1unuM+r7/Ufj2b2r77OWSf2ZurPGx74ICLSXIn03EcAec65jc65SuBV4Ip6Za4AXvBuvwl8w9rrtHntaNyltacTvvDko5l061n85LwTePaGr/Dd4QN4+eaR5Bx5OF8fnM3D14QOjlh4xwWR5/w5qiefYXBy3+4svvNCAAb06sLLN4/kS3278+7PzoqUC58IbcypfZpV184dMrnz8pNZ9JsL+OzeS5vfWBFpM+2xt57IOvd+QPSRBfnAyHhlnHOHzKwEOBJIr0uTtNKPzz2BS7/ch+lrCvnRWQNjlsnMMJ67sXZ5ZXa3Tiz49TfolJVJjy4dGHNqH8oqDnFU98NiPh/gy/168Oj3h7K3vIqzB/VmY/F+zj0pm5xxk+ndtSPFZZVcNaw/byzK57bzT+QrA3vx+Z4DHJ/dNXLZupdvHsGw40IfoPAa405ZGfTs0oFzT8rm9dx8Hv/BMJxzXHxKnzpL1XKO7MJmb5Jp8x8vi7sUcPovz+XVBVt5es4mIDRXscw7Re83hhzFUd07NTi3x4s3jaC8srrRozCjTbr1a3zz4cbXWreEmS5hJ6nz1RPb/oy11tRJ6s3sKuBi59zN3v3rgBHOuZ9FlVnllcn37m/wyuyq91pjgbEAxx577LAtW1p3VrkvqoqqajpmZtRZsx/9GBA5IVKiZq4t5JRjukfOz5G/5wBV1TUcn92VkvIqSg5U8fH6Ii4/tS+fbCima6cszht8FABlBw/RtVMWhfsq+GhdEd/9Su2Jrz7fe4BMM47u3okdJRWRE15VVdewZdd+OnfMYunWvYw5tU/kkOyyg4f4aF0hl592DACbi/cz9qVczht8FB+s3snvvnkKm4rKuHbkcUxesZ2enTty3uBs3ltZwI6SCr4zvD9z84opKKlg3sbd3H7hILIyMliydQ8VVdV1VkSFlVZUsa6glOKySjplZfCL15eyt7yKV8eO4pO8YtbsKGV9YSlfyenFsb26cNlpfRl45OEs2baHFz/dwsai/bxw0wi6dMzk/dU7+eoJR1J+sJpjj+xCVXUNe/ZXUuNCJwabtqqAY3p2ZuLifK4deSwLN+/h95NX87PzB/GvZdvZ6C3Lm3zbWbwyfyv/mL+V7w0fwC8uPIlR94WOjrzo5KPp0jGTt5fWLuu77RuDuGpYf95fvZOLTj6az/ce4IwBPVmwaTfLtu3lw3WFFO47yMiBvcju3onDO2bR/bAsunTK4ldRl2D874sHRw626dezM+PHDOGXry2jsrqGX10ymOXbSnhvVQH3XHEK909bV+fAodsvGMSYU/vyi9eWsmr7PsZdOoTV2/cxadl2fnTWQP7t9GP41iOfMPjobqzbWcqQPt044aiuTF6+g5EDe9U5t/9/XXQS/Y7ozC9eW8ZXTzgycorg84ccxYdrC3nuhq9w4/OhVStdOmbWOSL27EG9ueiUPvTo3IGJi/P5aF3DqyQlqmNWBlcO7cf6nWUM6duNl+fVTsAO7H04m4r3c9flJ3Pd6OMY+2Iuo44/kku+3IdZ64sbXPSmY1YG5w8+ioevPbPOGWybw8wWOeeGN1kugXAfDfzOOXexd388gHPuvqgy07wyn5pZFlAAZLtGXnz48OEuN7fxgxVERKSuRMM9ka+OhcAgMxtoZh2Bq4FJ9cpMAn7o3f4O8GFjwS4iIm2ryTF3bwz9VmAakAk865xbZWZ3A7nOuUnAM8BLZpYH7Cb0BSAiIimS0InDnHNTgCn1tt0VdbsCuCq5VRMRkZbSEaoiIgGkcBcRCSCFu4hIACncRUQCSOEuIhJATR7E1GZvbFYEtPQQ1d4E59QGakt6CkpbgtIOUFvCjnPOZTdVKGXh3hpmlpvIEVp+oLakp6C0JSjtALWluTQsIyISQAp3EZEA8mu4P5nqCiSR2pKegtKWoLQD1JZm8eWYu4iINM6vPXcREWmE78K9qYt1pwMze9bMCs1sZdS2Xmb2gZmt934f4W03M3vIa89yMxsa9ZwfeuXXm9kPY71XG7djgJnNNLM1ZrbKzH7u47YcZmYLzGyZ15b/9bYP9C7qvt67yHtHb3vci76b2Xhv+zozu7g64GWbAAAD2UlEQVS92+LVIdPMlpjZuz5vx2YzW2FmS80s19vmu8+XV4eeZvamma31/mZGp7Qtzjnf/BA65fAG4HigI7AMODnV9YpRz3OAocDKqG33A+O82+OAP3m3xwBTAQNGAfO97b2Ajd7vI7zbR7RzO/oCQ73b3YDPCF0k3Y9tMaCrd7sDMN+r4+vA1d72x4GfeLdvAR73bl8NvObdPtn73HUCBnqfx8wUfMZ+CfwDeNe779d2bAZ619vmu8+XV48XgJu92x2BnqlsS7s2Pgn/eKOBaVH3xwPjU12vOHXNoW64rwP6erf7Auu8208A19QvB1wDPBG1vU65FLXpHeBCv7cF6AIsJnQt4GIgq/7ni9D1C0Z7t7O8clb/Mxddrh3r3x+YAZwPvOvVy3ft8N53Mw3D3XefL6A7sAlvHjMd2uK3YZlYF+vul6K6NNfRzrkdAN7vo7zt8dqUVm31dufPJNTj9WVbvKGMpUAh8AGh3upe51z4IqDR9apz0XcgfNH3dGjLg8CvgBrv/pH4sx0ADnjfzBZZ6BrL4M/P1/FAEfCcN1z2tJkdTgrb4rdwb3hF6NCHw8/itSlt2mpmXYG3gNudc/saKxpjW9q0xTlX7Zw7g1DPdwTwpVjFvN9p2RYzuxwodM4tit4co2hatyPK15xzQ4FLgZ+a2TmNlE3ntmQRGop9zDl3JrCf0DBMPG3eFr+Fez4wIOp+f2B7nLLpZqeZ9QXwfhd62+O1KS3aamYdCAX7K865id5mX7YlzDm3F/iI0FhnTwtd1L1+vSJ19h7vQegSkqluy9eAb5rZZuBVQkMzD+K/dgDgnNvu/S4E/knoS9ePn698IN85N9+7/yahsE9ZW/wW7olcrDtdRV9E/IeExq/D26/3Zs9HASXe7ts04CIzO8KbYb/I29ZuzMwIXR93jXPur1EP+bEt2WbW07vdGbgAWAPMJHRRd2jYllgXfZ8EXO2tQhkIDAIWtE8rwDk33jnX3zmXQ+jz/6Fz7vv4rB0AZna4mXUL3yb0uViJDz9fzrkCYJuZDfY2fQNYTSrb0t4TKEmYuBhDaNXGBuCOVNcnTh0nADuAKkLfxD8iNM45A1jv/e7llTXgEa89K4DhUa9zE5Dn/dyYgnacRWiXcDmw1PsZ49O2nAYs8dqyErjL2348oVDLA94AOnnbD/Pu53mPHx/1Wnd4bVwHXJrCz9l51K6W8V07vDov835Whf+e/fj58upwBpDrfcbeJrTaJWVt0RGqIiIB5LdhGRERSYDCXUQkgBTuIiIBpHAXEQkghbuISAAp3EVEAkjhLiISQAp3EZEA+v80LpDFmB3luQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the FNN Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar to training the nerual network, we also need to load batches of test images and collect the outputs. The differences are that:\n",
    "* No loss & weights calculation\n",
    "* No wights update\n",
    "* Has correct prediction calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10K test images: 97 %\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "for images, labels in test_loader:\n",
    "    images = torch.FloatTensor(images.view(-1, 28*28))\n",
    "    \n",
    "    images = images\n",
    "    labels = labels\n",
    "    \n",
    "    outputs = net(images)\n",
    "    _, predicted = torch.max(outputs.data, 1)  # Choose the best class from the output: The class with the best score\n",
    "    total += labels.size(0)                    # Increment the total count\n",
    "    correct += (predicted == labels).sum()     # Increment the correct count\n",
    "    \n",
    "print('Accuracy of the network on the 10K test images: %d %%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the trained FNN Model for future use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model\n",
    "torch.save(net.state_dict(), 'fnn_model.pkl')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}