CIFAR_dropin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale = 1\n",
    "scale_size = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=4,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# functions to show an image\n",
    "\n",
    "\n",
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# get some random training images\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "# # show images\n",
    "# imshow(torchvision.utils.make_grid(images))\n",
    "# # print labels\n",
    "# print(' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, scale_size):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, int(120/scale_size))\n",
    "        self.fc2 = nn.Linear(int(120/scale_size), int(84/scale_size))\n",
    "        self.fc3 = nn.Linear(int(84/scale_size), 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "net = Net(scale_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = list()\n",
    "for i, param in enumerate(net.parameters()):\n",
    "    if i<4:\n",
    "        m = np.ones((param.detach().numpy()).shape)\n",
    "    else:\n",
    "        m = np.random.binomial(1, 1/scale, size=(param.detach().numpy()).shape)\n",
    "    mask.append(torch.tensor(m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  2000] loss: 2.295\n",
      "[1,  4000] loss: 2.066\n",
      "[1,  6000] loss: 1.813\n",
      "[1,  8000] loss: 1.690\n",
      "[1, 10000] loss: 1.552\n",
      "[1, 12000] loss: 1.538\n",
      "[2,  2000] loss: 1.459\n",
      "[2,  4000] loss: 1.409\n",
      "[2,  6000] loss: 1.373\n",
      "[2,  8000] loss: 1.377\n",
      "[2, 10000] loss: 1.321\n",
      "[2, 12000] loss: 1.309\n",
      "[3,  2000] loss: 1.247\n",
      "[3,  4000] loss: 1.209\n",
      "[3,  6000] loss: 1.221\n",
      "[3,  8000] loss: 1.221\n",
      "[3, 10000] loss: 1.214\n",
      "[3, 12000] loss: 1.201\n",
      "[4,  2000] loss: 1.110\n",
      "[4,  4000] loss: 1.119\n",
      "[4,  6000] loss: 1.120\n",
      "[4,  8000] loss: 1.108\n",
      "[4, 10000] loss: 1.120\n",
      "[4, 12000] loss: 1.101\n",
      "[5,  2000] loss: 1.016\n",
      "[5,  4000] loss: 1.048\n",
      "[5,  6000] loss: 1.025\n",
      "[5,  8000] loss: 1.053\n",
      "[5, 10000] loss: 1.039\n",
      "[5, 12000] loss: 1.040\n",
      "[6,  2000] loss: 0.940\n",
      "[6,  4000] loss: 0.977\n",
      "[6,  6000] loss: 0.966\n",
      "[6,  8000] loss: 0.989\n",
      "[6, 10000] loss: 0.983\n",
      "[6, 12000] loss: 0.997\n",
      "[7,  2000] loss: 0.894\n",
      "[7,  4000] loss: 0.917\n",
      "[7,  6000] loss: 0.932\n",
      "[7,  8000] loss: 0.942\n",
      "[7, 10000] loss: 0.926\n",
      "[7, 12000] loss: 0.932\n",
      "[8,  2000] loss: 0.829\n",
      "[8,  4000] loss: 0.862\n",
      "[8,  6000] loss: 0.886\n",
      "[8,  8000] loss: 0.897\n",
      "[8, 10000] loss: 0.905\n",
      "[8, 12000] loss: 0.908\n",
      "[9,  2000] loss: 0.791\n",
      "[9,  4000] loss: 0.826\n",
      "[9,  6000] loss: 0.858\n",
      "[9,  8000] loss: 0.845\n",
      "[9, 10000] loss: 0.869\n",
      "[9, 12000] loss: 0.873\n",
      "[10,  2000] loss: 0.766\n",
      "[10,  4000] loss: 0.817\n",
      "[10,  6000] loss: 0.811\n",
      "[10,  8000] loss: 0.818\n",
      "[10, 10000] loss: 0.835\n",
      "[10, 12000] loss: 0.842\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "losses = list()\n",
    "\n",
    "\n",
    "for epoch in range(10):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        \n",
    "        for p, m in zip(net.parameters(), mask):\n",
    "            p.grad *= m\n",
    "        \n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / 2000))\n",
    "            losses.append(running_loss / 2000)\n",
    "            running_loss = 0.0\n",
    "\n",
    "\n",
    "            \n",
    "print('Finished Training')\n",
    "\n",
    "np.save('loss_hist/losses_'+str(int(scale))+'_'+str(scale_size)+'.npy', np.array(losses))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = net(inputs)\n",
    "loss = criterion(outputs, labels)\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(inputs.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "grads = []\n",
    "for param in net.parameters():\n",
    "    grads.append(param.grad.view(-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "paras = []\n",
    "for param in net.parameters():\n",
    "    paras.append(param.view(-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = './cifar_net_'+str(int(scale))+'_'+str(scale_size)+'.pth'\n",
    "torch.save(net.state_dict(), PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "# # print images\n",
    "# imshow(torchvision.utils.make_grid(images))\n",
    "# print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = Net(scale_size)\n",
    "net.load_state_dict(torch.load(PATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = net(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted:    cat  ship  ship plane\n"
     ]
    }
   ],
   "source": [
    "_, predicted = torch.max(outputs, 1)\n",
    "\n",
    "print('Predicted: ', ' '.join('%5s' % classes[predicted[j]]\n",
    "                              for j in range(4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 64 %\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
    "    100 * correct / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of plane : 66 %\n",
      "Accuracy of   car : 80 %\n",
      "Accuracy of  bird : 54 %\n",
      "Accuracy of   cat : 45 %\n",
      "Accuracy of  deer : 47 %\n",
      "Accuracy of   dog : 49 %\n",
      "Accuracy of  frog : 79 %\n",
      "Accuracy of horse : 70 %\n",
      "Accuracy of  ship : 80 %\n",
      "Accuracy of truck : 70 %\n"
     ]
    }
   ],
   "source": [
    "class_correct = list(0. for i in range(10))\n",
    "class_total = list(0. for i in range(10))\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        c = (predicted == labels).squeeze()\n",
    "        for i in range(4):\n",
    "            label = labels[i]\n",
    "            class_correct[label] += c[i].item()\n",
    "            class_total[label] += 1\n",
    "\n",
    "\n",
    "for i in range(10):\n",
    "    print('Accuracy of %5s : %2d %%' % (\n",
    "        classes[i], 100 * class_correct[i] / class_total[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "import pickle\n",
    "\n",
    "paras = list()\n",
    "dists = list()\n",
    "for param in net.parameters():\n",
    "    paras.append((param.view(-1)).detach().numpy())\n",
    "    weights = np.squeeze(paras[-1].flatten())\n",
    "    dists.append(stats.gaussian_kde(weights))\n",
    "    \n",
    "pickle.dump( dists, open( \"dists.pkl\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for p, d in zip(paras, dists):\n",
    "#     x_plot = np.linspace(p.min(), p.max(), 100, endpoint=True)\n",
    "#     plt.figure()\n",
    "#     plt.hist(p, bins=100, density=True)\n",
    "#     plt.plot(x_plot, d.pdf(x_plot))\n",
    "#     plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}