03 - 1. CNN with Mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一般的mnist和一般的cnn(print parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2)\n",
    "        self.fc = nn.Linear(320, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.mp(F.relu(self.conv1(x)))\n",
    "        x = self.mp(F.relu(self.conv2(x)))\n",
    "        x = x.view(in_size, -1)  # flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "\n",
    "\n",
    "model = Net()\n",
    "\n",
    "total_params = sum(p.numel() for p in model.parameters())\n",
    "print(f'{total_params:,} total parameters.')\n",
    "total_trainable_params = sum(\n",
    "    p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f'{total_trainable_params:,} training parameters.')\n",
    "\n",
    "\n",
    "\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPU Enabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2)\n",
    "        self.fc = nn.Linear(320, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.mp(F.relu(self.conv1(x)))\n",
    "        x = self.mp(F.relu(self.conv2(x)))\n",
    "        x = x.view(in_size, -1)  # flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "\n",
    "model = Net()\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        # sum up batch loss\n",
    "        #test_loss += F.nll_loss(output, target, reduction='sum').item()\n",
    "        test_loss += criterion(output, target).item()\n",
    "        # get the index of the max log-probability\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加了L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2)\n",
    "        self.fc = nn.Linear(320, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.mp(F.relu(self.conv1(x)))\n",
    "        x = self.mp(F.relu(self.conv2(x)))\n",
    "        x = x.view(in_size, -1)  # flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "\n",
    "\n",
    "model = Net()\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        regularization_loss = 0\n",
    "        for param in model.parameters():\n",
    "            regularization_loss += torch.mean(abs(param))\n",
    "        \n",
    "        loss = criterion(output, target)\n",
    "        loss += 0.01 * regularization_loss # lamda = 0.01\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        # sum up batch loss\n",
    "        #test_loss += F.nll_loss(output, target, reduction='sum').item()\n",
    "        test_loss += criterion(output, target).item()\n",
    "        # get the index of the max log-probability\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加了L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2)\n",
    "        self.fc = nn.Linear(320, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.mp(F.relu(self.conv1(x)))\n",
    "        x = self.mp(F.relu(self.conv2(x)))\n",
    "        x = x.view(in_size, -1)  # flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "\n",
    "model = Net()\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalization Input Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "trans = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=trans,\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=trans)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2)\n",
    "        self.fc = nn.Linear(320, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.mp(F.relu(self.conv1(x)))\n",
    "        x = self.mp(F.relu(self.conv2(x)))\n",
    "        x = x.view(in_size, -1)  # flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "model = Net()\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With Batch Norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "trans = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=trans,\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=trans)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv1_bn = nn.BatchNorm2d(10)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_bn = nn.BatchNorm2d(20)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc1_bn = nn.BatchNorm1d(50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "        self.fc2_bn = nn.BatchNorm1d(10)\n",
    "\n",
    "    def forward(self, x, y=None):\n",
    "        x = F.relu(F.max_pool2d(self.conv1_bn(self.conv1(x)), 2))\n",
    "        x = F.max_pool2d(self.conv2_drop(self.conv2_bn(self.conv2(x))), 2)\n",
    "        x = F.relu(x)\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1_bn(self.fc1(x)))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2_bn(self.fc2(x))\n",
    "        return F.log_softmax(x, dim=0)\n",
    "model = Net()\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 31):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 64\n",
    "\n",
    "trans = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=trans,\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=trans)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv1_bn = nn.BatchNorm2d(10)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_bn = nn.BatchNorm2d(20)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc1_bn = nn.BatchNorm1d(50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "        self.fc2_bn = nn.BatchNorm1d(10)\n",
    "\n",
    "    def forward(self, x, y=None):\n",
    "        x = F.relu(F.max_pool2d(self.conv1_bn(self.conv1(x)), 2))\n",
    "        x = F.max_pool2d(self.conv2_drop(self.conv2_bn(self.conv2(x))), 2)\n",
    "        x = F.relu(x)\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1_bn(self.fc1(x)))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2_bn(self.fc2(x))\n",
    "        return F.log_softmax(x, dim=0)\n",
    "\n",
    "model = Net()\n",
    "if torch.cuda.device_count() > 2:\n",
    "    print(\"On\", torch.cuda.device_count(), \"GPUs\")\n",
    "    model = nn.DataParallel(model)\n",
    "\n",
    "model = model.to(device)\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.001)\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 31):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 換上預選的model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.22it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.63it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1:\tTest set: Average loss: 0.0001, Accuracy: 9836/10000 (98%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.21it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.64it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2:\tTest set: Average loss: 0.0001, Accuracy: 9892/10000 (98%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.18it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.65it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3:\tTest set: Average loss: 0.0001, Accuracy: 9909/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.19it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.66it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4:\tTest set: Average loss: 0.0001, Accuracy: 9925/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:34<00:00,  4.15it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.60it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5:\tTest set: Average loss: 0.0001, Accuracy: 9924/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.21it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.66it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6:\tTest set: Average loss: 0.0001, Accuracy: 9918/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.20it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.61it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7:\tTest set: Average loss: 0.0001, Accuracy: 9922/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.19it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.60it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8:\tTest set: Average loss: 0.0001, Accuracy: 9923/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.17it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.63it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9:\tTest set: Average loss: 0.0001, Accuracy: 9928/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.22it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.65it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:\tTest set: Average loss: 0.0001, Accuracy: 9931/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.22it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.66it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:\tTest set: Average loss: 0.0001, Accuracy: 9929/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.21it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.61it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12:\tTest set: Average loss: 0.0001, Accuracy: 9929/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.18it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.64it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13:\tTest set: Average loss: 0.0001, Accuracy: 9934/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.22it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.65it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14:\tTest set: Average loss: 0.0001, Accuracy: 9929/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.20it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.60it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:\tTest set: Average loss: 0.0001, Accuracy: 9933/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.11it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.65it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16:\tTest set: Average loss: 0.0001, Accuracy: 9928/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.21it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.62it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17:\tTest set: Average loss: 0.0001, Accuracy: 9930/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.20it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.63it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18:\tTest set: Average loss: 0.0001, Accuracy: 9930/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.19it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.63it/s]\n",
      "  0%|          | 0/118 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19:\tTest set: Average loss: 0.0001, Accuracy: 9930/10000 (99%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  4.21it/s]\n",
      "100%|██████████| 20/20 [00:03<00:00,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:\tTest set: Average loss: 0.0001, Accuracy: 9926/10000 (99%)\n",
      "done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms, models\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Check if cuda available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Training settings\n",
    "batch_size = 512\n",
    "\n",
    "trans = transforms.Compose([\n",
    "            transforms.Grayscale(num_output_channels=3),\n",
    "            transforms.Resize(32),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/',\n",
    "                               train=True,\n",
    "                               transform=trans,\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='./data/',\n",
    "                              train=False,\n",
    "                              transform=trans)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)\n",
    "\n",
    "num_classes = 10\n",
    "\n",
    "model = models.densenet201(pretrained=True)\n",
    "num_ftrs = model.classifier.in_features\n",
    "model.classifier = nn.Linear(num_ftrs, 10)\n",
    "\n",
    "if torch.cuda.device_count() > 1:\n",
    "    print(\"On\", torch.cuda.device_count(), \"GPUs\")\n",
    "    model = nn.DataParallel(model)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5, weight_decay=0.001)\n",
    "\n",
    "\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(tqdm(train_loader)):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in tqdm(test_loader):\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).item()\n",
    "        pred = output.data.max(1, keepdim=True)[1]\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print(str(epoch)+\":\\t\"+'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "for epoch in range(1, 21):\n",
    "    train(epoch)\n",
    "    test(epoch)\n",
    "print('done')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch]",
   "language": "python",
   "name": "conda-env-pytorch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}