learning-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = torch.double\n",
    "device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# yhat, y: (num_examples, num_outputs)\n",
    "# return: (num_examples,)\n",
    "def log_cross_entropy(yhat, y):\n",
    "#     return - torch.sum(y * torch.log(yhat+1e-6))\n",
    "    return - yhat[y > 0].log().sum()\n",
    "\n",
    "def cross_entropy(yhat, y):\n",
    "    return - yhat[y > 0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# W: (num_inputs, num_outputs)\n",
    "# b: (num_outputs,)\n",
    "# x: (num_examples, num_inputs)\n",
    "# return: (num_examples, num_outputs)\n",
    "def linear_transform(W, b, x):\n",
    "    return torch.mm(x, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# z: (num_examples, num_outputs)\n",
    "# return: (num_examples, num_outputs)\n",
    "def softmax(z):\n",
    "    z_min, _ = torch.min(z, dim=1)\n",
    "    z = z - z_min.unsqueeze(1)\n",
    "    exp = torch.exp(z)\n",
    "    sum_exp = torch.sum(exp, dim=1).unsqueeze(1)\n",
    "    return exp / sum_exp\n",
    "\n",
    "def log_softmax(z):\n",
    "    z_mean = torch.mean(z, dim=1)\n",
    "    z = z - z_mean.unsqueeze(1)\n",
    "    exp = torch.exp(z)\n",
    "    sum_exp = torch.sum(exp, dim=1).unsqueeze(1)\n",
    "    return z - sum_exp.log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset import MNIST\n",
    "\n",
    "train_data = MNIST('data/mnist/train-images-idx3-ubyte.gz', 'data/mnist/train-labels-idx1-ubyte.gz')\n",
    "test_data = MNIST('data/mnist/t10k-images-idx3-ubyte.gz', 'data/mnist/t10k-labels-idx1-ubyte.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.input_dim, train_data.target_dim, train_data.sample_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_examples = train_data.sample_size\n",
    "num_inputs = np.prod(train_data.input_dim)\n",
    "num_outputs = np.prod(train_data.target_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W = torch.randn(num_inputs, num_outputs, device=device, dtype=dtype, requires_grad=True)\n",
    "b = torch.randn(num_outputs, device=device, dtype=dtype, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common import split_data\n",
    "\n",
    "batch_size = 64\n",
    "epochs = 10\n",
    "learning_rate = .002\n",
    "\n",
    "for e in range(epochs):\n",
    "    cumulative_cross_entroy = 0\n",
    "    for i, (input_batch, target_batch) in enumerate(split_data(train_data.inputs, train_data.targets, batch_size)):\n",
    "        bx = torch.from_numpy(input_batch.reshape(-1, num_inputs)).to(device)\n",
    "        by = torch.from_numpy(target_batch).to(device)\n",
    "        z = linear_transform(W, b, bx)\n",
    "        yhat = log_softmax(z)\n",
    "        loss = cross_entropy(yhat, by)\n",
    "        cumulative_cross_entroy += loss.item()\n",
    "        loss.backward()\n",
    "        with torch.no_grad():\n",
    "            W -= learning_rate * W.grad\n",
    "            b -= learning_rate * b.grad\n",
    "        W.grad.zero_()\n",
    "        b.grad.zero_()\n",
    "    print(cumulative_cross_entroy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_predict(W, b, x):\n",
    "    z = linear_transform(W, b, x)\n",
    "    yhat = softmax(z)\n",
    "    pred = torch.argmax(yhat, dim=1)\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_correct = 0.0\n",
    "num_total = len(test_data.inputs)\n",
    "for i, (input_batch, target_batch) in enumerate(split_data(test_data.inputs, test_data.targets, batch_size)):\n",
    "    tx = torch.from_numpy(input_batch.reshape(-1, num_inputs)).to(device)\n",
    "    ty = torch.from_numpy(target_batch).to(device)\n",
    "    pred = model_predict(W, b, tx)\n",
    "    label = torch.argmax(ty, axis=1)\n",
    "    num_correct += torch.sum(pred == label).item()\n",
    "print(num_correct / num_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    test_input = torch.from_numpy(test_data.inputs[np.newaxis, i].reshape(-1, num_inputs)).to(device)\n",
    "    pred = model_predict(W, b, test_input)\n",
    "    test_data.show_image(i, \"prediction %d\" % pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}