laplacian_eigenmaps_nn_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch, math, copy\n",
    "import numpy as np\n",
    "from torchvision import datasets, transforms\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import TensorDataset, DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ntargets = torch.zeros(10000, 1)\\nfor i, (data, target) in enumerate(test_loader):\\n    targets[i] = target\\n    # print(type(targets[i]))\\ntargets = DataLoader(TensorDataset(targets), batch_size=256, shuffle=False)\\n# print(len(targets))\\nfor target in targets:\\n    print(target[0].view(1, -1).shape)\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])\n",
    "\n",
    "# Replace with path to MNIST on your machine\n",
    "train_dataset = datasets.MNIST(\"/Users/lucastucker/REU-2023/archive/\", train=True, download=True, transform=transform)\n",
    "train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(\"/Users/lucastucker/REU-2023/archive/\", train=False, download=True, transform=transform)\n",
    "test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)\n",
    "\n",
    "\"\"\"\n",
    "targets = torch.zeros(10000, 1)\n",
    "for i, (data, target) in enumerate(test_loader):\n",
    "    targets[i] = target\n",
    "    # print(type(targets[i]))\n",
    "targets = DataLoader(TensorDataset(targets), batch_size=256, shuffle=False)\n",
    "# print(len(targets))\n",
    "for target in targets:\n",
    "    print(target[0].view(1, -1).shape)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epochs, model, criterion, optimizer, train_loader, test_loader, reduced_dim, t_nearest, batch_size):\n",
    "    for epoch in range(epochs):\n",
    "        train_err = train_epoch(model, criterion, optimizer, train_loader, reduced_dim, t_nearest, batch_size)\n",
    "        test_err = test(model, test_loader, reduced_dim, t_nearest, batch_size)\n",
    "        print('Epoch {:03d}/{:03d}, Train Error {:.2f}% || Test Error {:.2f}%'.format(epoch, epochs, train_err*100, test_err*100))\n",
    "    return train_err, test_err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(model, criterion, optimizer, loader, reduced_dim, t_nearest, batch_size):\n",
    "    total_correct = 0.\n",
    "    total_samples = 0.\n",
    "\n",
    "    data_matrix = np.zeros((60000, 784))\n",
    "    targets = torch.zeros((60000, 1))\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(loader):\n",
    "        data_matrix[batch_idx] = data.view(1, 784)    \n",
    "        targets[batch_idx] = target    \n",
    "\n",
    "    data_matrix = le_on_loader(data_matrix, reduced_dim ** 2, t_nearest, sigma=0.3)\n",
    "    # print(f\"data matrix looks like {data_matrix}\")\n",
    "    reduced_tensor = torch.Tensor(data_matrix).view(-1, 1, reduced_dim, reduced_dim)\n",
    "\n",
    "    targets = DataLoader(TensorDataset(torch.Tensor(targets)), batch_size=batch_size, shuffle=False)\n",
    "    dataloader = DataLoader(TensorDataset(reduced_tensor), batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    # You can now iterate over dataloader for mini-batch processing\n",
    "    for data, target in zip(dataloader, targets):\n",
    "        # NOTE: Uncomment the code below if you are using a GPU\n",
    "        # if torch.cuda.is_available():\n",
    "        #    data, target = data.cuda(), target.cuda()\n",
    "        print(f\"data[0] is of shape {data[0].shape}\")\n",
    "        print(f\"target[0] is of shape {target[0].shape}\")\n",
    "        print(f\"target[0].view(-1) has shape {target[0].view(-1).shape}\")\n",
    "\n",
    "        output = model(data[0])\n",
    "\n",
    "        # long vs float issue\n",
    "        loss = F.cross_entropy(output, target[0].view(-1).long())\n",
    "        preds = output.argmax(dim=1, keepdim=True)\n",
    "        total_correct += preds.eq(target[0].long().view_as(preds)).sum().item() # compare preds to target\n",
    "        total_samples += torch.numel(preds) # numel short for number of elements\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    return 1 - total_correct/total_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, loader, reduced_dim, t_nearest, batch_size):\n",
    "    total_correct = 0.\n",
    "    total_samples = 0.\n",
    "    model.eval()\n",
    "\n",
    "    data_matrix = np.zeros((10000, 784))\n",
    "    targets = np.zeros((10000, 1))\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(loader):\n",
    "        data_matrix[batch_idx] = data.view(1, 784)    \n",
    "        targets[batch_idx] = target    \n",
    "\n",
    "    data_matrix = le_on_loader(data_matrix, reduced_dim ** 2, t_nearest, 0.3)\n",
    "    reduced_tensor = torch.Tensor(data_matrix).view(-1, 1, reduced_dim, reduced_dim)\n",
    "\n",
    "    # Create a DataLoader\n",
    "    targets = DataLoader(TensorDataset(torch.Tensor(targets)), batch_size=batch_size, shuffle=False)\n",
    "    dataloader = DataLoader(TensorDataset(reduced_tensor), batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for data, target in zip(dataloader, targets):\n",
    "            # NOTE: Uncomment the code below if you are using a GPU\n",
    "            # if torch.cuda.is_available():\n",
    "            #    data, target = data.cuda(), target.cuda()\n",
    "\n",
    "            output = model(data[0])\n",
    "            preds = output.argmax(dim=1, keepdim=True)\n",
    "            total_correct += preds.eq(target[0].long().view_as(preds)).sum().item()\n",
    "            total_samples += preds.numel()\n",
    "\n",
    "    return 1 - total_correct/total_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run /Users/lucastucker/REU-2023/laplacian_eigenmaps_functions.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def le_on_loader(X, n_components, t_neighbors, sigma):\n",
    "    return laplacian_eigenmaps(X, n_components, t_neighbors, sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNNeluBN(nn.Module):\n",
    "    def __init__(self, reduced_dim):\n",
    "        super(CNNeluBN, self).__init__()\n",
    "\n",
    "        # write code here to instantiate layers\n",
    "        # for example, self.conv = nn.Conv2d(1, 4, 3, 1, 1)\n",
    "        # creates a conv layer with 1 input channel, 4 output\n",
    "        # channels, a 3x3 kernel, and stride=padding=1\n",
    "        self.layers = nn.ModuleList()\n",
    "\n",
    "        self.layers.append(nn.Conv2d(1, 4, 3, 1, 1)) # 1 to 4 channels on 1 x red x red input\n",
    "        self.layers.append(nn.BatchNorm2d(4))\n",
    "        self.layers.append(nn.ELU())\n",
    "        self.layers.append(nn.AvgPool2d(2, 2)) # Now size is 4 x red // 2 x red // 2\n",
    "        self.layers.append(nn.Conv2d(4, 8, 3, 1, 1)) # 4 to 8 channels\n",
    "        self.layers.append(nn.BatchNorm2d(8))\n",
    "        self.layers.append(nn.ELU())\n",
    "        self.layers.append(nn.AvgPool2d(2, 2)) # Now size is 8 x red // 2 // 2 x red // 2 // 2\n",
    "        new_dim = (reduced_dim // 2) // 2\n",
    "        self.layers.append(nn.Linear(8 * new_dim * new_dim, 10))\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                sigma = 1 / (math.sqrt(9 * m.out_channels)) # 9 is k^2\n",
    "                m.weight.data.normal_(0, sigma)\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "    def forward(self, input):\n",
    "      # print(input.shape)\n",
    "      u = self.layers[0](input)\n",
    "      for layer in self.layers[1:-1]:\n",
    "         u = layer(u)\n",
    "      num_batches = u.size()[0]\n",
    "      u = u.view(num_batches, -1)\n",
    "      return self.layers[-1](u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss() # TODO (implement in nn) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training ELU CNN + BN with representation dimension 20 x 20\n",
      "train_loader\n",
      "called laplacian_eigenmaps!\n",
      "got nearest neighbors, distances, indices!\n",
      "Made W with shape (60000, 60000)\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "lr = 0.05\n",
    "reduced_dim = 20\n",
    "t_nearest = 50\n",
    "batch_size = 512\n",
    "print(f\"Training ELU CNN + BN with representation dimension {reduced_dim} x {reduced_dim}\")\n",
    "model = CNNeluBN(reduced_dim) # .cuda() \n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)\n",
    "print(f\"train_loader\")\n",
    "train_errs, test_errs = train(3, model, criterion, optimizer, train_loader, test_loader, reduced_dim, t_nearest, batch_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}