CNN_MNIST.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyOdI4G1QKJhDThCv7wRKaa5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bijouvj/ST-summer-2024/blob/main/CNN_MNIST.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "-qUY2oYGEBWO"
      },
      "outputs": [],
      "source": [
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.datasets import fetch_openml\n",
        "mnist = fetch_openml('mnist_784', version=1, cache=True, as_frame=False)"
      ],
      "metadata": {
        "id": "U6AntubpEL9r"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.utils.data\n",
        "from torch.autograd import Variable\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "w-BemT6-EbPU"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mnist.target = mnist.target.astype(np.int8) # fetch_openml() returns targets as strings"
      ],
      "metadata": {
        "id": "WSQJnrVzFh3t"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X, y = mnist[\"data\"], mnist[\"target\"]"
      ],
      "metadata": {
        "id": "ruheDk0zETxS"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)"
      ],
      "metadata": {
        "id": "qR0RUVMbEoD2"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(y_train.dtype)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rVjjtUTREsyw",
        "outputId": "c8f09fa7-c660-4acd-e5c3-c5210c79793b"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "int8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 32\n",
        "\n",
        "torch_X_train = torch.from_numpy(X_train).type(torch.LongTensor)\n",
        "torch_y_train = torch.from_numpy(y_train).type(torch.LongTensor) # data type is long\n",
        "\n",
        "# create feature and targets tensor for test set.\n",
        "torch_X_test = torch.from_numpy(X_test).type(torch.LongTensor)\n",
        "torch_y_test = torch.from_numpy(y_test).type(torch.LongTensor) # data type is long\n",
        "\n",
        "# Pytorch train and test sets\n",
        "train = torch.utils.data.TensorDataset(torch_X_train,torch_y_train)\n",
        "test = torch.utils.data.TensorDataset(torch_X_test,torch_y_test)\n",
        "\n",
        "# data loader\n",
        "train_loader = torch.utils.data.DataLoader(train, batch_size = BATCH_SIZE, shuffle = False)\n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size = BATCH_SIZE, shuffle = False)"
      ],
      "metadata": {
        "id": "c_V0GMtqEup5"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MLP(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MLP, self).__init__()\n",
        "        self.linear1 = nn.Linear(784,250)\n",
        "        self.linear2 = nn.Linear(250,100)\n",
        "        self.linear3 = nn.Linear(100,10)\n",
        "\n",
        "    def forward(self,X):\n",
        "        X = F.relu(self.linear1(X))\n",
        "        X = F.relu(self.linear2(X))\n",
        "        X = self.linear3(X)\n",
        "        return F.log_softmax(X, dim=1)\n",
        "\n",
        "mlp = MLP()\n",
        "print(mlp)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9wvHfPeNE7q1",
        "outputId": "7383c5d3-558c-451d-9039-b5090cbe3535"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MLP(\n",
            "  (linear1): Linear(in_features=784, out_features=250, bias=True)\n",
            "  (linear2): Linear(in_features=250, out_features=100, bias=True)\n",
            "  (linear3): Linear(in_features=100, out_features=10, bias=True)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def fit(model, train_loader):\n",
        "    optimizer = torch.optim.Adam(model.parameters())#,lr=0.001, betas=(0.9,0.999))\n",
        "    error = nn.CrossEntropyLoss()\n",
        "    EPOCHS = 5\n",
        "    model.train()\n",
        "    for epoch in range(EPOCHS):\n",
        "        correct = 0\n",
        "        for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n",
        "            var_X_batch = Variable(X_batch).float()\n",
        "            var_y_batch = Variable(y_batch)\n",
        "            optimizer.zero_grad()\n",
        "            output = model(var_X_batch)\n",
        "            loss = error(output, var_y_batch)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # Total correct predictions\n",
        "            predicted = torch.max(output.data, 1)[1]\n",
        "            correct += (predicted == var_y_batch).sum()\n",
        "            #print(correct)\n",
        "            if batch_idx % 50 == 0:\n",
        "                print(f'Epoch : {epoch} \\tLoss: {loss.item():.6f}\\t Accuracy:{float(BATCH_SIZE*(batch_idx+1)):.3f}')"
      ],
      "metadata": {
        "id": "b-_7k_cdFtfq"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fit(mlp, train_loader)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "tGt3Ur_oFzY2",
        "outputId": "c06f648e-e006-4510-cc8a-a01a672ae003"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch : 0 \tLoss: 4.528050\t Accuracy:32.000\n",
            "Epoch : 0 \tLoss: 0.763592\t Accuracy:1632.000\n",
            "Epoch : 0 \tLoss: 0.165692\t Accuracy:3232.000\n",
            "Epoch : 0 \tLoss: 0.419522\t Accuracy:4832.000\n",
            "Epoch : 0 \tLoss: 0.225305\t Accuracy:6432.000\n",
            "Epoch : 0 \tLoss: 0.310715\t Accuracy:8032.000\n",
            "Epoch : 0 \tLoss: 0.525159\t Accuracy:9632.000\n",
            "Epoch : 0 \tLoss: 0.209523\t Accuracy:11232.000\n",
            "Epoch : 0 \tLoss: 0.181472\t Accuracy:12832.000\n",
            "Epoch : 0 \tLoss: 0.215006\t Accuracy:14432.000\n",
            "Epoch : 0 \tLoss: 0.326348\t Accuracy:16032.000\n",
            "Epoch : 0 \tLoss: 0.101830\t Accuracy:17632.000\n",
            "Epoch : 0 \tLoss: 0.357150\t Accuracy:19232.000\n",
            "Epoch : 0 \tLoss: 0.315405\t Accuracy:20832.000\n",
            "Epoch : 0 \tLoss: 0.264099\t Accuracy:22432.000\n",
            "Epoch : 0 \tLoss: 0.330311\t Accuracy:24032.000\n",
            "Epoch : 0 \tLoss: 0.101897\t Accuracy:25632.000\n",
            "Epoch : 0 \tLoss: 0.034556\t Accuracy:27232.000\n",
            "Epoch : 0 \tLoss: 0.101703\t Accuracy:28832.000\n",
            "Epoch : 0 \tLoss: 0.104981\t Accuracy:30432.000\n",
            "Epoch : 0 \tLoss: 0.271280\t Accuracy:32032.000\n",
            "Epoch : 0 \tLoss: 0.094192\t Accuracy:33632.000\n",
            "Epoch : 0 \tLoss: 0.083355\t Accuracy:35232.000\n",
            "Epoch : 0 \tLoss: 0.316269\t Accuracy:36832.000\n",
            "Epoch : 0 \tLoss: 0.560608\t Accuracy:38432.000\n",
            "Epoch : 0 \tLoss: 0.335980\t Accuracy:40032.000\n",
            "Epoch : 0 \tLoss: 0.821782\t Accuracy:41632.000\n",
            "Epoch : 0 \tLoss: 0.139272\t Accuracy:43232.000\n",
            "Epoch : 0 \tLoss: 0.202263\t Accuracy:44832.000\n",
            "Epoch : 0 \tLoss: 0.129432\t Accuracy:46432.000\n",
            "Epoch : 0 \tLoss: 0.183414\t Accuracy:48032.000\n",
            "Epoch : 0 \tLoss: 0.084093\t Accuracy:49632.000\n",
            "Epoch : 0 \tLoss: 0.356816\t Accuracy:51232.000\n",
            "Epoch : 0 \tLoss: 0.192436\t Accuracy:52832.000\n",
            "Epoch : 0 \tLoss: 0.087585\t Accuracy:54432.000\n",
            "Epoch : 1 \tLoss: 0.233962\t Accuracy:32.000\n",
            "Epoch : 1 \tLoss: 0.049625\t Accuracy:1632.000\n",
            "Epoch : 1 \tLoss: 0.067982\t Accuracy:3232.000\n",
            "Epoch : 1 \tLoss: 0.194164\t Accuracy:4832.000\n",
            "Epoch : 1 \tLoss: 0.090822\t Accuracy:6432.000\n",
            "Epoch : 1 \tLoss: 0.216472\t Accuracy:8032.000\n",
            "Epoch : 1 \tLoss: 0.123437\t Accuracy:9632.000\n",
            "Epoch : 1 \tLoss: 0.418172\t Accuracy:11232.000\n",
            "Epoch : 1 \tLoss: 0.103395\t Accuracy:12832.000\n",
            "Epoch : 1 \tLoss: 0.015116\t Accuracy:14432.000\n",
            "Epoch : 1 \tLoss: 0.100153\t Accuracy:16032.000\n",
            "Epoch : 1 \tLoss: 0.071916\t Accuracy:17632.000\n",
            "Epoch : 1 \tLoss: 0.031179\t Accuracy:19232.000\n",
            "Epoch : 1 \tLoss: 0.072006\t Accuracy:20832.000\n",
            "Epoch : 1 \tLoss: 0.146868\t Accuracy:22432.000\n",
            "Epoch : 1 \tLoss: 0.267907\t Accuracy:24032.000\n",
            "Epoch : 1 \tLoss: 0.049278\t Accuracy:25632.000\n",
            "Epoch : 1 \tLoss: 0.019633\t Accuracy:27232.000\n",
            "Epoch : 1 \tLoss: 0.155056\t Accuracy:28832.000\n",
            "Epoch : 1 \tLoss: 0.028960\t Accuracy:30432.000\n",
            "Epoch : 1 \tLoss: 0.128529\t Accuracy:32032.000\n",
            "Epoch : 1 \tLoss: 0.077721\t Accuracy:33632.000\n",
            "Epoch : 1 \tLoss: 0.108606\t Accuracy:35232.000\n",
            "Epoch : 1 \tLoss: 0.200713\t Accuracy:36832.000\n",
            "Epoch : 1 \tLoss: 0.121336\t Accuracy:38432.000\n",
            "Epoch : 1 \tLoss: 0.253366\t Accuracy:40032.000\n",
            "Epoch : 1 \tLoss: 0.314743\t Accuracy:41632.000\n",
            "Epoch : 1 \tLoss: 0.275947\t Accuracy:43232.000\n",
            "Epoch : 1 \tLoss: 0.034484\t Accuracy:44832.000\n",
            "Epoch : 1 \tLoss: 0.034535\t Accuracy:46432.000\n",
            "Epoch : 1 \tLoss: 0.053698\t Accuracy:48032.000\n",
            "Epoch : 1 \tLoss: 0.093746\t Accuracy:49632.000\n",
            "Epoch : 1 \tLoss: 0.115692\t Accuracy:51232.000\n",
            "Epoch : 1 \tLoss: 0.012689\t Accuracy:52832.000\n",
            "Epoch : 1 \tLoss: 0.165618\t Accuracy:54432.000\n",
            "Epoch : 2 \tLoss: 0.118487\t Accuracy:32.000\n",
            "Epoch : 2 \tLoss: 0.153492\t Accuracy:1632.000\n",
            "Epoch : 2 \tLoss: 0.034608\t Accuracy:3232.000\n",
            "Epoch : 2 \tLoss: 0.053957\t Accuracy:4832.000\n",
            "Epoch : 2 \tLoss: 0.200895\t Accuracy:6432.000\n",
            "Epoch : 2 \tLoss: 0.409915\t Accuracy:8032.000\n",
            "Epoch : 2 \tLoss: 0.127737\t Accuracy:9632.000\n",
            "Epoch : 2 \tLoss: 0.710683\t Accuracy:11232.000\n",
            "Epoch : 2 \tLoss: 0.101380\t Accuracy:12832.000\n",
            "Epoch : 2 \tLoss: 0.027843\t Accuracy:14432.000\n",
            "Epoch : 2 \tLoss: 0.210880\t Accuracy:16032.000\n",
            "Epoch : 2 \tLoss: 0.045674\t Accuracy:17632.000\n",
            "Epoch : 2 \tLoss: 0.017152\t Accuracy:19232.000\n",
            "Epoch : 2 \tLoss: 0.239585\t Accuracy:20832.000\n",
            "Epoch : 2 \tLoss: 0.290750\t Accuracy:22432.000\n",
            "Epoch : 2 \tLoss: 0.286423\t Accuracy:24032.000\n",
            "Epoch : 2 \tLoss: 0.135595\t Accuracy:25632.000\n",
            "Epoch : 2 \tLoss: 0.047238\t Accuracy:27232.000\n",
            "Epoch : 2 \tLoss: 0.037042\t Accuracy:28832.000\n",
            "Epoch : 2 \tLoss: 0.109453\t Accuracy:30432.000\n",
            "Epoch : 2 \tLoss: 0.228118\t Accuracy:32032.000\n",
            "Epoch : 2 \tLoss: 0.192002\t Accuracy:33632.000\n",
            "Epoch : 2 \tLoss: 0.028364\t Accuracy:35232.000\n",
            "Epoch : 2 \tLoss: 0.092186\t Accuracy:36832.000\n",
            "Epoch : 2 \tLoss: 0.172406\t Accuracy:38432.000\n",
            "Epoch : 2 \tLoss: 0.103445\t Accuracy:40032.000\n",
            "Epoch : 2 \tLoss: 0.323710\t Accuracy:41632.000\n",
            "Epoch : 2 \tLoss: 0.043283\t Accuracy:43232.000\n",
            "Epoch : 2 \tLoss: 0.055540\t Accuracy:44832.000\n",
            "Epoch : 2 \tLoss: 0.054280\t Accuracy:46432.000\n",
            "Epoch : 2 \tLoss: 0.217594\t Accuracy:48032.000\n",
            "Epoch : 2 \tLoss: 0.259778\t Accuracy:49632.000\n",
            "Epoch : 2 \tLoss: 0.121919\t Accuracy:51232.000\n",
            "Epoch : 2 \tLoss: 0.029038\t Accuracy:52832.000\n",
            "Epoch : 2 \tLoss: 0.068066\t Accuracy:54432.000\n",
            "Epoch : 3 \tLoss: 0.189421\t Accuracy:32.000\n",
            "Epoch : 3 \tLoss: 0.036209\t Accuracy:1632.000\n",
            "Epoch : 3 \tLoss: 0.110122\t Accuracy:3232.000\n",
            "Epoch : 3 \tLoss: 0.032543\t Accuracy:4832.000\n",
            "Epoch : 3 \tLoss: 0.080064\t Accuracy:6432.000\n",
            "Epoch : 3 \tLoss: 0.284658\t Accuracy:8032.000\n",
            "Epoch : 3 \tLoss: 0.056228\t Accuracy:9632.000\n",
            "Epoch : 3 \tLoss: 0.306714\t Accuracy:11232.000\n",
            "Epoch : 3 \tLoss: 0.147760\t Accuracy:12832.000\n",
            "Epoch : 3 \tLoss: 0.005408\t Accuracy:14432.000\n",
            "Epoch : 3 \tLoss: 0.300513\t Accuracy:16032.000\n",
            "Epoch : 3 \tLoss: 0.065285\t Accuracy:17632.000\n",
            "Epoch : 3 \tLoss: 0.025403\t Accuracy:19232.000\n",
            "Epoch : 3 \tLoss: 0.019711\t Accuracy:20832.000\n",
            "Epoch : 3 \tLoss: 0.543773\t Accuracy:22432.000\n",
            "Epoch : 3 \tLoss: 0.108822\t Accuracy:24032.000\n",
            "Epoch : 3 \tLoss: 0.133275\t Accuracy:25632.000\n",
            "Epoch : 3 \tLoss: 0.054619\t Accuracy:27232.000\n",
            "Epoch : 3 \tLoss: 0.074408\t Accuracy:28832.000\n",
            "Epoch : 3 \tLoss: 0.011243\t Accuracy:30432.000\n",
            "Epoch : 3 \tLoss: 0.078750\t Accuracy:32032.000\n",
            "Epoch : 3 \tLoss: 0.000879\t Accuracy:33632.000\n",
            "Epoch : 3 \tLoss: 0.053229\t Accuracy:35232.000\n",
            "Epoch : 3 \tLoss: 0.037464\t Accuracy:36832.000\n",
            "Epoch : 3 \tLoss: 0.168323\t Accuracy:38432.000\n",
            "Epoch : 3 \tLoss: 0.080642\t Accuracy:40032.000\n",
            "Epoch : 3 \tLoss: 0.702898\t Accuracy:41632.000\n",
            "Epoch : 3 \tLoss: 0.093659\t Accuracy:43232.000\n",
            "Epoch : 3 \tLoss: 0.097174\t Accuracy:44832.000\n",
            "Epoch : 3 \tLoss: 0.113676\t Accuracy:46432.000\n",
            "Epoch : 3 \tLoss: 0.008153\t Accuracy:48032.000\n",
            "Epoch : 3 \tLoss: 0.038463\t Accuracy:49632.000\n",
            "Epoch : 3 \tLoss: 0.076345\t Accuracy:51232.000\n",
            "Epoch : 3 \tLoss: 0.132286\t Accuracy:52832.000\n",
            "Epoch : 3 \tLoss: 0.103608\t Accuracy:54432.000\n",
            "Epoch : 4 \tLoss: 0.131773\t Accuracy:32.000\n",
            "Epoch : 4 \tLoss: 0.014869\t Accuracy:1632.000\n",
            "Epoch : 4 \tLoss: 0.004624\t Accuracy:3232.000\n",
            "Epoch : 4 \tLoss: 0.041105\t Accuracy:4832.000\n",
            "Epoch : 4 \tLoss: 0.223730\t Accuracy:6432.000\n",
            "Epoch : 4 \tLoss: 0.100626\t Accuracy:8032.000\n",
            "Epoch : 4 \tLoss: 0.017392\t Accuracy:9632.000\n",
            "Epoch : 4 \tLoss: 0.246593\t Accuracy:11232.000\n",
            "Epoch : 4 \tLoss: 0.091163\t Accuracy:12832.000\n",
            "Epoch : 4 \tLoss: 0.045650\t Accuracy:14432.000\n",
            "Epoch : 4 \tLoss: 0.115468\t Accuracy:16032.000\n",
            "Epoch : 4 \tLoss: 0.034219\t Accuracy:17632.000\n",
            "Epoch : 4 \tLoss: 0.069887\t Accuracy:19232.000\n",
            "Epoch : 4 \tLoss: 0.072014\t Accuracy:20832.000\n",
            "Epoch : 4 \tLoss: 0.330341\t Accuracy:22432.000\n",
            "Epoch : 4 \tLoss: 0.124100\t Accuracy:24032.000\n",
            "Epoch : 4 \tLoss: 0.121160\t Accuracy:25632.000\n",
            "Epoch : 4 \tLoss: 0.060468\t Accuracy:27232.000\n",
            "Epoch : 4 \tLoss: 0.071445\t Accuracy:28832.000\n",
            "Epoch : 4 \tLoss: 0.022039\t Accuracy:30432.000\n",
            "Epoch : 4 \tLoss: 0.215252\t Accuracy:32032.000\n",
            "Epoch : 4 \tLoss: 0.004384\t Accuracy:33632.000\n",
            "Epoch : 4 \tLoss: 0.131677\t Accuracy:35232.000\n",
            "Epoch : 4 \tLoss: 0.067625\t Accuracy:36832.000\n",
            "Epoch : 4 \tLoss: 0.282260\t Accuracy:38432.000\n",
            "Epoch : 4 \tLoss: 0.084118\t Accuracy:40032.000\n",
            "Epoch : 4 \tLoss: 0.338871\t Accuracy:41632.000\n",
            "Epoch : 4 \tLoss: 0.025920\t Accuracy:43232.000\n",
            "Epoch : 4 \tLoss: 0.042355\t Accuracy:44832.000\n",
            "Epoch : 4 \tLoss: 0.020447\t Accuracy:46432.000\n",
            "Epoch : 4 \tLoss: 0.199715\t Accuracy:48032.000\n",
            "Epoch : 4 \tLoss: 0.001008\t Accuracy:49632.000\n",
            "Epoch : 4 \tLoss: 0.135848\t Accuracy:51232.000\n",
            "Epoch : 4 \tLoss: 0.044256\t Accuracy:52832.000\n",
            "Epoch : 4 \tLoss: 0.017881\t Accuracy:54432.000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model):\n",
        "#model = mlp\n",
        "    correct = 0\n",
        "    for test_imgs, test_labels in test_loader:\n",
        "        #print(test_imgs.shape)\n",
        "        test_imgs = Variable(test_imgs).float()\n",
        "        output = model(test_imgs)\n",
        "        predicted = torch.max(output,1)[1]\n",
        "        correct += (predicted == test_labels).sum()\n",
        "    print(\"Test accuracy:{:.3f}% \".format( float(correct) / (len(test_loader)*BATCH_SIZE)))\n",
        "\n",
        "evaluate(mlp)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bu47ofvLF1qR",
        "outputId": "47db836c-003a-48d0-c4c5-5e26ac980e7d"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test accuracy:0.959% \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Explanation¶\n",
        "\n",
        "To better understand convolutional neural network I recommend the great section on it here : http://cs231n.github.io/convolutional-networks/\n",
        "\n",
        "**Convolutional operation** : First let's clarify briefly how we can perform the convolutional operation on an image. For that we need to define a kernel which is a small matrix of size 5 * 5 for example. To perform the convolution operation, we just need to slide the kernel along the image horizontally and vertically and do the dot product of the kernel and the small portion of the image.\n",
        "\n",
        "**Pooling** : the convolutional operation give an output of the same size of the input image. To reduce the size of the image and thus reduce the number of paramers in the model we perform a Pooling operation. The pooling operation need a window size.. By sliding the window along the image, we compute the mean or the max of the portion of the image inside the window in case of MeanPooling or MaxPooling.\n",
        "\n",
        "**Stride** is the number of pixels to pass at a time when sliding the convolutional kernel.\n",
        "\n",
        "**Padding** to preserve exactly the size of the input image, it is useful to add a zero padding on the border of the image.\n",
        "\n",
        "To remember : What makes a CNN so interesting for images is that it is invariant by translation and for each convolutional layer we only need to store the kernels. Thus we can stack a lot of layers to learn deep features without having too much parameters that would make a model untrainnable.\n"
      ],
      "metadata": {
        "id": "jSAWhsBdHyVK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch_X_train = torch_X_train.view(-1, 1,28,28).float()\n",
        "torch_X_test = torch_X_test.view(-1,1,28,28).float()\n",
        "print(torch_X_train.shape)\n",
        "print(torch_X_test.shape)\n",
        "\n",
        "# Pytorch train and test sets\n",
        "train = torch.utils.data.TensorDataset(torch_X_train,torch_y_train)\n",
        "test = torch.utils.data.TensorDataset(torch_X_test,torch_y_test)\n",
        "\n",
        "# data loader\n",
        "train_loader = torch.utils.data.DataLoader(train, batch_size = BATCH_SIZE, shuffle = False)\n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size = BATCH_SIZE, shuffle = False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d7eTf9diHqfY",
        "outputId": "7a6a3b12-b4f7-4c8d-8843-85b311d6b2a0"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([56000, 1, 28, 28])\n",
            "torch.Size([14000, 1, 28, 28])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class CNN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(CNN, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(1, 32, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(32, 32, kernel_size=5)\n",
        "        self.conv3 = nn.Conv2d(32,64, kernel_size=5)\n",
        "        self.fc1 = nn.Linear(3*3*64, 256)\n",
        "        self.fc2 = nn.Linear(256, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.conv1(x))\n",
        "        #x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = F.relu(F.max_pool2d(self.conv2(x), 2))\n",
        "        x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = F.relu(F.max_pool2d(self.conv3(x),2))\n",
        "        x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = x.view(-1,3*3*64 )\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        return F.log_softmax(x, dim=1)\n",
        "\n",
        "cnn = CNN()\n",
        "print(cnn)\n",
        "\n",
        "it = iter(train_loader)\n",
        "X_batch, y_batch = next(it)\n",
        "print(cnn.forward(X_batch).shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I2OqL_HUH-SD",
        "outputId": "b0368d42-fc0a-407b-cd49-b068c526ce75"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CNN(\n",
            "  (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1))\n",
            "  (conv2): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1))\n",
            "  (conv3): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1))\n",
            "  (fc1): Linear(in_features=576, out_features=256, bias=True)\n",
            "  (fc2): Linear(in_features=256, out_features=10, bias=True)\n",
            ")\n",
            "torch.Size([32, 10])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fit(cnn,train_loader)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 649
        },
        "id": "LccAicR-ICLa",
        "outputId": "06ca3c65-bce7-47b8-eeed-284061053d33"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch : 0 \tLoss: 18.033966\t Accuracy:32.000\n",
            "Epoch : 0 \tLoss: 2.215352\t Accuracy:1632.000\n",
            "Epoch : 0 \tLoss: 1.279084\t Accuracy:3232.000\n",
            "Epoch : 0 \tLoss: 1.138644\t Accuracy:4832.000\n",
            "Epoch : 0 \tLoss: 0.552001\t Accuracy:6432.000\n",
            "Epoch : 0 \tLoss: 0.752180\t Accuracy:8032.000\n",
            "Epoch : 0 \tLoss: 0.441516\t Accuracy:9632.000\n",
            "Epoch : 0 \tLoss: 0.377081\t Accuracy:11232.000\n",
            "Epoch : 0 \tLoss: 0.235730\t Accuracy:12832.000\n",
            "Epoch : 0 \tLoss: 0.384197\t Accuracy:14432.000\n",
            "Epoch : 0 \tLoss: 0.566501\t Accuracy:16032.000\n",
            "Epoch : 0 \tLoss: 0.308301\t Accuracy:17632.000\n",
            "Epoch : 0 \tLoss: 0.235839\t Accuracy:19232.000\n",
            "Epoch : 0 \tLoss: 0.276869\t Accuracy:20832.000\n",
            "Epoch : 0 \tLoss: 0.828639\t Accuracy:22432.000\n",
            "Epoch : 0 \tLoss: 0.338092\t Accuracy:24032.000\n",
            "Epoch : 0 \tLoss: 0.380655\t Accuracy:25632.000\n",
            "Epoch : 0 \tLoss: 0.256919\t Accuracy:27232.000\n",
            "Epoch : 0 \tLoss: 0.099698\t Accuracy:28832.000\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-1c26c3cd6f27>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcnn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-24-b1afc98d32a7>\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, train_loader)\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvar_X_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_y_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    523\u001b[0m                 \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m             )\n\u001b[0;32m--> 525\u001b[0;31m         torch.autograd.backward(\n\u001b[0m\u001b[1;32m    526\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m     \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 267\u001b[0;31m     _engine_run_backward(\n\u001b[0m\u001b[1;32m    268\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m         \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py\u001b[0m in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    742\u001b[0m         \u001b[0munregister_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_register_logging_hooks_on_whole_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    743\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 744\u001b[0;31m         return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    745\u001b[0m             \u001b[0mt_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    746\u001b[0m         )  # Calls into the C++ engine to run the backward pass\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate(cnn)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZPoo-tQdIGWg",
        "outputId": "11937bde-c628-4bde-8585-54349f16b0c5"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test accuracy:0.914% \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "8wL2kgbmIUzX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}