test copy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:48.244049Z",
     "start_time": "2021-12-09T07:56:35.814143Z"
    }
   },
   "outputs": [],
   "source": [
    "from math import pi\n",
    "from math import cos\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "\n",
    "import visdom\n",
    "import numpy as np\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "import pickle\n",
    "\n",
    "import copy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.485580Z",
     "start_time": "2021-12-09T07:56:48.245050Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "cuda = torch.cuda.is_available()\n",
    "batch_size = 64\n",
    "\n",
    "# load data\n",
    "transform = transforms.Compose([transforms.ToTensor(),\n",
    "                                transforms.Normalize(\n",
    "                                    (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
    "                                ])\n",
    "\n",
    "train = datasets.CIFAR10('data/cifar10', train=True, download=True,\n",
    "                         transform=transform)\n",
    "test = datasets.CIFAR10('data/cifar10', train=False, transform=transform)\n",
    "\n",
    "p = 0.9\n",
    "train_size = int(p*len(train))\n",
    "val_size = len(train)-train_size\n",
    "\n",
    "torch.manual_seed(0)\n",
    "train, val = random_split(train, [train_size, val_size])\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train, batch_size=batch_size, shuffle=True)\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "    val, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    test, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "data_loaders = {\"train\": train_loader, \"val\": val_loader}\n",
    "data_lengths = {\"train\": train_size, \"val\": val_size}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.501594Z",
     "start_time": "2021-12-09T07:56:50.486579Z"
    }
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32,\n",
    "                               kernel_size=5,\n",
    "                               stride=1)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3)\n",
    "        self.conv2_bn = nn.BatchNorm2d(64)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3)\n",
    "        self.conv3_bn = nn.BatchNorm2d(64)\n",
    "        self.dense1 = nn.Linear(in_features=4 * 64, out_features=128)\n",
    "        self.dense1_bn = nn.BatchNorm1d(128)\n",
    "        self.dense2 = nn.Linear(128, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_bn(self.conv2(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv3_bn(self.conv3(x)), 2))\n",
    "        x = x.view(-1, 4 * 64)\n",
    "        x = F.relu(self.dense1_bn(self.dense1(x)))\n",
    "        x = F.log_softmax(self.dense2(x), dim=1)  # NLL loss expects log_softmax\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.517597Z",
     "start_time": "2021-12-09T07:56:50.502594Z"
    }
   },
   "outputs": [],
   "source": [
    "def proposed_lr(initial_lr, iteration, epoch_per_cycle):\n",
    "    # proposed learning late function\n",
    "    return initial_lr * (cos(pi * iteration / epoch_per_cycle) + 1) / 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proposed_lr_new(initial_lr, minimum_lr, iteration, epoch_per_cycle):\n",
    "    # proposed learning late function\n",
    "    return minimum_lr+(initial_lr-minimum_lr) * (cos(pi * iteration / epoch_per_cycle) + 1) / 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.533601Z",
     "start_time": "2021-12-09T07:56:50.518597Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_se(model, epochs, cycles, initial_lr, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    _lr_list, _loss_list = [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "\n",
    "        for j in range(epochs_per_cycle):\n",
    "            _epoch_loss = 0\n",
    "\n",
    "            for batch_idx, (data, target) in enumerate(train_loader):\n",
    "                lr = proposed_lr(initial_lr, j+batch_idx /\n",
    "                                 len(train_loader), epochs_per_cycle)\n",
    "                optimizer.param_groups[0]['lr'] = lr\n",
    "                if cuda:\n",
    "                    data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                loss = F.nll_loss(output, target)\n",
    "                _epoch_loss += loss.data/len(train_loader)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "            _loss_list.append(_epoch_loss.cpu())\n",
    "            count += 1\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr\",\n",
    "                         opts=dict(title=\"learning rate\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "    return snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.533601Z",
     "start_time": "2021-12-09T07:56:50.518597Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_se_per_epoch(model, epochs, cycles, initial_lr, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    _lr_list, _loss_list = [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "\n",
    "        for j in range(epochs_per_cycle):\n",
    "            _epoch_loss = 0\n",
    "\n",
    "            lr = proposed_lr(initial_lr, j, epochs_per_cycle)\n",
    "            optimizer.param_groups[0]['lr'] = lr\n",
    "\n",
    "            for batch_idx, (data, target) in enumerate(train_loader):\n",
    "                if cuda:\n",
    "                    data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                loss = F.nll_loss(output, target)\n",
    "                _epoch_loss += loss.data/len(train_loader)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "            _loss_list.append(_epoch_loss.cpu())\n",
    "            count += 1\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr (epoch)\",\n",
    "                         opts=dict(title=\"learning rate\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (epoch)\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log) (epoch)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "    return snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.549605Z",
     "start_time": "2021-12-09T07:56:50.535603Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_se_valloss_per_epoch(model, epochs, cycles, initial_lr, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    loss_snapshots = []\n",
    "    valloss_snapshots = []\n",
    "    _lr_list, _loss_list, _valloss_list = [], [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "\n",
    "        for j in range(epochs_per_cycle):\n",
    "\n",
    "            for phase in ['train', 'val']:\n",
    "                if phase == 'train':\n",
    "                    lr = proposed_lr(initial_lr, j, epochs_per_cycle)\n",
    "                    optimizer.param_groups[0]['lr'] = lr\n",
    "                    model.train(True)  # Set model to training mode\n",
    "                else:\n",
    "                    model.train(False)  # Set model to evaluate mode\n",
    "\n",
    "                _epoch_loss = 0\n",
    "\n",
    "\n",
    "                for batch_idx, (data, target) in enumerate(data_loaders[phase]):\n",
    "                    if cuda:\n",
    "                        data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                    optimizer.zero_grad()\n",
    "                    output = model(data)\n",
    "                    loss = F.nll_loss(output, target)\n",
    "                    _epoch_loss += loss.data/len(data_loaders[phase])\n",
    "\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                if phase == 'train':\n",
    "                    _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "                    _loss_list.append(_epoch_loss.cpu())\n",
    "                    count += 1\n",
    "                else:\n",
    "                    _valloss_list.append(_epoch_loss.cpu())\n",
    "\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr (epoch)\",\n",
    "                        opts=dict(title=\"learning rate\",\n",
    "                                xlabel=\"epochs\",\n",
    "                                ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (epoch)\",\n",
    "                        opts=dict(title=\"loss\",\n",
    "                                xlabel=\"epochs\",\n",
    "                                ylabel=\"training loss (s.e.)\"))\n",
    "                vis.line(np.array(_valloss_list), np.arange(count),  win=\"loss (epoch)\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"validation loss (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log) (epoch)\",\n",
    "                        opts=dict(title=\"loss (log)\",\n",
    "                                xlabel=\"epochs\",\n",
    "                                ylabel=\"training loss (s.e.)\",\n",
    "                                ytype=\"log\"))\n",
    "                vis.line(np.array(_valloss_list), np.arange(count),  win=\"loss (log) (epoch)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"validation loss (s.e.)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "        loss_snapshots.append(_loss_list[-1])\n",
    "        valloss_snapshots.append(_valloss_list[-1])\n",
    "    return snapshots, loss_snapshots, valloss_snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.549605Z",
     "start_time": "2021-12-09T07:56:50.535603Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_se_valloss(model, epochs, cycles, initial_lr, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    loss_snapshots = []\n",
    "    valloss_snapshots = []\n",
    "    _lr_list, _loss_list, _valloss_list = [], [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "\n",
    "        for j in range(epochs_per_cycle):\n",
    "\n",
    "            for phase in ['train', 'val']:\n",
    "                if phase == 'train':\n",
    "                    model.train(True)  # Set model to training mode\n",
    "                else:\n",
    "                    model.train(False)  # Set model to evaluate mode\n",
    "\n",
    "                _epoch_loss = 0\n",
    "\n",
    "                for batch_idx, (data, target) in enumerate(data_loaders[phase]):\n",
    "                    if phase == 'train':\n",
    "                        lr = proposed_lr(initial_lr, j+batch_idx /\n",
    "                                         len(data_loaders[phase]), epochs_per_cycle)\n",
    "                        optimizer.param_groups[0]['lr'] = lr\n",
    "                    if cuda:\n",
    "                        data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                    optimizer.zero_grad()\n",
    "                    output = model(data)\n",
    "                    loss = F.nll_loss(output, target)\n",
    "                    _epoch_loss += loss.data/len(data_loaders[phase])\n",
    "\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                if phase == 'train':\n",
    "                    _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "                    _loss_list.append(_epoch_loss.cpu())\n",
    "                    count += 1\n",
    "                else:\n",
    "                    _valloss_list.append(_epoch_loss.cpu())\n",
    "\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr\",\n",
    "                        opts=dict(title=\"learning rate\",\n",
    "                                xlabel=\"epochs\",\n",
    "                                ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss\", name=\"loss\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"loss (s.e.)\"))\n",
    "                vis.line(np.array(_valloss_list), np.arange(count),  win=\"loss\", update='append', name=\"valloss\")\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log)\", name=\"loss\",\n",
    "                        opts=dict(title=\"loss (log)\",\n",
    "                                xlabel=\"epochs\",\n",
    "                                ylabel=\"loss (s.e.)\",\n",
    "                                ytype=\"log\"))\n",
    "                vis.line(np.array(_valloss_list), np.arange(count),  win=\"loss (log)\", update='append', name=\"valloss\")\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "        loss_snapshots.append(_loss_list[-1])\n",
    "        valloss_snapshots.append(_valloss_list[-1])\n",
    "    return snapshots, loss_snapshots, valloss_snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'test'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vis.line(\n",
    "    X=np.arange(1, 38),\n",
    "    Y=np.random.randn(37),\n",
    "    win=\"test\",\n",
    "    name='6',\n",
    "    update='append',\n",
    ")\n",
    "\n",
    "vis.line(\n",
    "    X=np.arange(1, 38),\n",
    "    Y=np.random.randn(37),\n",
    "    win=\"test\",\n",
    "    name='11',\n",
    "    update='append',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_se_loss(model, epochs, cycles, initial_lr, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    loss_snapshots = []\n",
    "    _lr_list, _loss_list = [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "\n",
    "        for j in range(epochs_per_cycle):\n",
    "            _epoch_loss = 0\n",
    "\n",
    "            for batch_idx, (data, target) in enumerate(train_loader):\n",
    "                lr = proposed_lr(initial_lr, j+batch_idx /\n",
    "                                 len(train_loader), epochs_per_cycle)\n",
    "                optimizer.param_groups[0]['lr'] = lr\n",
    "                if cuda:\n",
    "                    data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                loss = F.nll_loss(output, target)\n",
    "                _epoch_loss += loss.data/len(train_loader)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "            _loss_list.append(_epoch_loss.cpu())\n",
    "            count += 1\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr\",\n",
    "                         opts=dict(title=\"learning rate\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "        loss_snapshots.append(_loss_list[-1])\n",
    "    return snapshots, loss_snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.549605Z",
     "start_time": "2021-12-09T07:56:50.535603Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_se_loss_lr(model, epochs, cycles, initial_lr, burnin = 1, vis=None):\n",
    "    \"\"\"\n",
    "    during an iteration a batch goes forward and backward  \n",
    "    while during an epoch every batch of a data set is processed\n",
    "    \"\"\"\n",
    "    snapshots = []\n",
    "    loss_snapshots = []\n",
    "    _lr_list, _loss_list = [], []\n",
    "    count = 0\n",
    "    epochs_per_cycle = epochs // cycles\n",
    "    optimizer = optim.SGD(model.parameters(), lr=initial_lr)\n",
    "\n",
    "    for i in range(cycles):\n",
    "        for j in range(epochs_per_cycle):\n",
    "            _epoch_loss = 0\n",
    "\n",
    "            for batch_idx, (data, target) in enumerate(train_loader):\n",
    "                if i > burnin-1:\n",
    "                    lr = proposed_lr_new(\n",
    "                        initial_lr, initial_lr/100, j+batch_idx/len(train_loader), epochs_per_cycle)\n",
    "                    optimizer.param_groups[0]['lr'] = lr\n",
    "                if cuda:\n",
    "                    data, target = data.cuda(), target.cuda()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                loss = F.nll_loss(output, target)\n",
    "                _epoch_loss += loss.data/len(train_loader)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "            _loss_list.append(_epoch_loss.cpu())\n",
    "            count += 1\n",
    "\n",
    "            if vis is not None and j % 5 == 4:\n",
    "                vis.line(np.array(_lr_list), np.arange(count), win=\"lr (new update)\",\n",
    "                         opts=dict(title=\"learning rate\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"learning rate (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (new update)\",\n",
    "                         opts=dict(title=\"loss\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\"))\n",
    "                vis.line(np.array(_loss_list), np.arange(count),  win=\"loss (log) (new update)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (s.e.)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "        snapshots.append(copy.deepcopy(model.state_dict()))\n",
    "        loss_snapshots.append(_loss_list[-1])\n",
    "    return snapshots, loss_snapshots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.581031Z",
     "start_time": "2021-12-09T07:56:50.566608Z"
    }
   },
   "outputs": [],
   "source": [
    "def test_se(Model, snapshots, use_model_num):\n",
    "    index = len(snapshots) - use_model_num\n",
    "    snapshots = snapshots[index:]\n",
    "    model_list = [Model() for _ in snapshots]\n",
    "\n",
    "    for model, weight in zip(model_list, snapshots):\n",
    "        model.load_state_dict(weight)\n",
    "        model.eval()\n",
    "        if cuda:\n",
    "            model.cuda()\n",
    "\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        if cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        output_list = [model(data).unsqueeze(0) for model in model_list]\n",
    "        output = torch.mean(torch.cat(output_list), 0).squeeze()\n",
    "        test_loss += F.nll_loss(output, target).data\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.eq(target.data).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100 * correct / len(test_loader.dataset)))\n",
    "\n",
    "    return test_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.596033Z",
     "start_time": "2021-12-09T07:56:50.582031Z"
    }
   },
   "outputs": [],
   "source": [
    "def test_se_loss(Model, snapshots, loss_snapshots, use_model_num):\n",
    "    # use the models with least losses\n",
    "    loss_snapshots = np.array(loss_snapshots)\n",
    "    index = np.argpartition(loss_snapshots, use_model_num)[:use_model_num]\n",
    "    snapshots = [snapshots[i] for i in index]\n",
    "    model_list = [Model() for _ in snapshots]\n",
    "\n",
    "    for model, weight in zip(model_list, snapshots):\n",
    "        model.load_state_dict(weight)\n",
    "        model.eval()\n",
    "        if cuda:\n",
    "            model.cuda()\n",
    "\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        if cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        output_list = [model(data).unsqueeze(0) for model in model_list]\n",
    "        output = torch.mean(torch.cat(output_list), 0).squeeze()\n",
    "        test_loss += F.nll_loss(output, target).data\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.eq(target.data).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100 * correct / len(test_loader.dataset)))\n",
    "\n",
    "    return test_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.612037Z",
     "start_time": "2021-12-09T07:56:50.597034Z"
    }
   },
   "outputs": [],
   "source": [
    "def test_se_vote(Model, snapshots, use_model_num):\n",
    "    index = len(snapshots) - use_model_num\n",
    "    snapshots = snapshots[index:]\n",
    "    model_list = [Model() for _ in snapshots]\n",
    "\n",
    "    for model, weight in zip(model_list, snapshots):\n",
    "        model.load_state_dict(weight)\n",
    "        model.eval()\n",
    "        if cuda:\n",
    "            model.cuda()\n",
    "\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        if cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        output_list = [model(data).unsqueeze(0) for model in model_list]\n",
    "        output = torch.mean(torch.cat(output_list), 0).squeeze()\n",
    "        test_loss += F.nll_loss(output, target).data\n",
    "        pred_list = torch.cat([item.data.max(2)[1] for item in output_list])\n",
    "        pred = torch.mode(pred_list,0)[0]  # most vote\n",
    "        correct += pred.eq(target.data).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100 * correct / len(test_loader.dataset)))\n",
    "\n",
    "    return test_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.628045Z",
     "start_time": "2021-12-09T07:56:50.613037Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_normal(model, epochs, vis=None):\n",
    "\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    _lr_list, _loss_list = [], []\n",
    "    for epoch in range(epochs):\n",
    "        _epoch_loss = 0\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            if cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            output = model(data)\n",
    "            loss = F.nll_loss(output, target)\n",
    "            _epoch_loss += loss.data / len(train_loader)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        _loss_list.append(_epoch_loss.cpu())\n",
    "        _lr_list.append(optimizer.param_groups[0]['lr'])\n",
    "\n",
    "        if vis is not None and epoch % 5 == 4:\n",
    "            vis.line(np.array(_lr_list), np.arange(epoch+1), win=\"lr_n\",\n",
    "                     opts=dict(title=\"learning rate\",\n",
    "                               xlabel=\"epochs\",\n",
    "                               ylabel=\"learning rate (normal)\"))\n",
    "            vis.line(np.array(_loss_list), np.arange(epoch+1), win=\"loss_n\",\n",
    "                     opts=dict(title=\"loss\",\n",
    "                               xlabel=\"epochs\",\n",
    "                               ylabel=\"training loss (normal)\"))\n",
    "            vis.line(np.array(_loss_list), np.arange(epoch+1),  win=\"loss_n (log)\",\n",
    "                         opts=dict(title=\"loss (log)\",\n",
    "                                   xlabel=\"epochs\",\n",
    "                                   ylabel=\"training loss (normal)\",\n",
    "                                   ytype=\"log\"))\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T07:56:50.644057Z",
     "start_time": "2021-12-09T07:56:50.629046Z"
    }
   },
   "outputs": [],
   "source": [
    "def test_normal(model):\n",
    "\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        if cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        output = model(data)\n",
    "        test_loss += F.nll_loss(output, target).data\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.eq(target.data).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100 * correct / len(test_loader.dataset)))\n",
    "\n",
    "    return test_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T08:49:39.513476Z",
     "start_time": "2021-12-09T07:56:50.645057Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting up a new session...\n"
     ]
    }
   ],
   "source": [
    "vis = visdom.Visdom(port=6006)\n",
    "\n",
    "model1 = Net()\n",
    "if cuda:\n",
    "    model1.cuda()\n",
    "\n",
    "models1, losses1, vallosses1 = train_se_valloss(model1, 300, 12, 0.2, vis)\n",
    "pickle.dump((models1,losses1,vallosses1), open(\"save/se-val-original-300-12-0.2-2.p\", \"wb\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-09T08:49:39.514476Z",
     "start_time": "2021-12-09T08:49:39.514476Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "snapshot ensemble (original)\n",
      "\n",
      "Test set: Average loss: 2.1052, Accuracy: 7590/10000 (76%)\n",
      "\n",
      "---\n",
      "snapshot ensemble (min loss)\n",
      "\n",
      "Test set: Average loss: 2.1061, Accuracy: 7582/10000 (76%)\n",
      "\n",
      "---\n",
      "snapshot ensemble (vote)\n",
      "\n",
      "Test set: Average loss: 2.1148, Accuracy: 7574/10000 (76%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(2.1148, device='cuda:0')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"snapshot ensemble (original)\")\n",
    "test_se(Net, models1, 5)\n",
    "print(\"---\")\n",
    "print(\"snapshot ensemble (min loss)\")\n",
    "test_se_loss(Net, models1, losses1, 5)\n",
    "print(\"---\")\n",
    "print(\"snapshot ensemble (min valloss)\")\n",
    "test_se_loss(Net, models1, vallosses1, 5)\n",
    "print(\"---\")\n",
    "print(\"snapshot ensemble (vote)\")\n",
    "test_se_vote(Net, models1, 5)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "fd034ed54560f0698c2946b7ca675e493afbd7ee3c0ecf162ae3deac3cf4477b"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}