NetMLPLowRank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import utils\n",
    "from nets.NetMLPLowRank import NetMLPLowRank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "train_loader, test_loader = utils.build_dataset(dataset='MNIST')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if batch_idx % 200 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item()))\n",
    "            \n",
    "def test(model, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            output = model(data)\n",
    "            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss\n",
    "            pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  3.64k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.436297\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 1.478329\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 1.438827\n",
      "\n",
      "Test set: Average loss: -3.0435, Accuracy: 3703/10000 (37%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[1], d=[1])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  6.79k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.408969\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.512498\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.561827\n",
      "\n",
      "Test set: Average loss: -5.7071, Accuracy: 8685/10000 (87%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[2], d=[2])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  10.95k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.403645\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.296845\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.256664\n",
      "\n",
      "Test set: Average loss: -7.3666, Accuracy: 9264/10000 (93%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[2], d=[4])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  19.27k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.299727\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.322407\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.184402\n",
      "\n",
      "Test set: Average loss: -7.5882, Accuracy: 9437/10000 (94%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[2], d=[8])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  11.0k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.326322\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.210988\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.213570\n",
      "\n",
      "Test set: Average loss: -6.6677, Accuracy: 9276/10000 (93%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[4], d=[2])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  19.32k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.346809\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.347984\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.227573\n",
      "\n",
      "Test set: Average loss: -7.1737, Accuracy: 9320/10000 (93%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[4], d=[4])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  35.96k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.360572\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.335885\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.307477\n",
      "\n",
      "Test set: Average loss: -6.5466, Accuracy: 9459/10000 (95%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[4], d=[8])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  19.43k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.361831\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.294236\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.187545\n",
      "\n",
      "Test set: Average loss: -6.6336, Accuracy: 9348/10000 (93%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[8], d=[2])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  36.07k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.354477\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.259531\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.214605\n",
      "\n",
      "Test set: Average loss: -6.8199, Accuracy: 9478/10000 (95%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[8], d=[4])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MLP | Params =  69.35k\n",
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.291168\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.250233\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.202087\n",
      "\n",
      "Test set: Average loss: -6.6493, Accuracy: 9402/10000 (94%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr = 0.01\n",
    "n_epochs = 100\n",
    "\n",
    "model = NetMLPLowRank(K=[8], d=[8])\n",
    "\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print (f'Model MLP | Params = ', utils.params_to_string(pytorch_total_params))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test(model, test_loader)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}