updated intro to NN

02_intro_neural_networks/01_introduction_mnist.ipynb

@@ -77,7 +77,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "training_data, validation_data = torch.utils.data.random_split(training_data, [0.8, 0.2], generator=torch.Generator().manual_seed(55))"
+    "train_size = int(0.8 * len(training_data))  # 80% for training\n",
+    "val_size = len(training_data) - train_size  # Remaining 20% for validation\n",
+    "training_data, validation_data = torch.utils.data.random_split(training_data, [train_size, val_size], generator=torch.Generator().manual_seed(55))\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n"
    ]
   },
   {
@@ -215,7 +228,7 @@
     "print(linear_model)\n",
     "\n",
     "loss_fn = nn.CrossEntropyLoss()\n",
-    "optimizer = torch.optim.SGD(linear_model.parameters(), lr=0.01)"
+    "optimizer = torch.optim.SGD(linear_model.parameters(), lr=0.05)"
    ]
   },
   {
@@ -300,12 +313,20 @@
     "%%time\n",
     "\n",
     "epochs = 5\n",
+    "train_loss_all = []\n",
+    "val_loss_all = []\n",
     "for j in range(epochs):\n",
     "    train_one_epoch(train_dataloader, linear_model, loss_fn, optimizer)\n",
     "    \n",
     "    # checking on the training loss and accuracy once per epoch\n",
     "    acc, loss = evaluate(train_dataloader, linear_model, loss_fn)\n",
-    "    print(f\"Epoch {j}: training loss: {loss}, accuracy: {acc}\")"
+    "    train_loss_all.append(loss)\n",
+    "    print(f\"Epoch {j}: training loss: {loss}, accuracy: {acc}\")\n",
+    "    \n",
+    "    # checking on the validation loss and accuracy once per epoch\n",
+    "    val_acc, val_loss = evaluate(val_dataloader, linear_model, loss_fn)\n",
+    "    val_loss_all.append(val_loss)\n",
+    "    print(f\"Epoch {j}: val. loss: {val_loss}, val. accuracy: {val_acc}\")"
    ]
   },
   {
@@ -339,9 +360,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Inference\n",
+    "# Prediction\n",
     "\n",
-    "For a better measure of the quality of the model, let's see the model accuracy for the validation data.\n",
+    "Let's see how our modek generalizes to the unseen test data.\n",
     "\n"
    ]
   },
@@ -351,8 +372,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "acc_val, loss_val = evaluate(val_dataloader, linear_model, loss_fn)\n",
-    "print(\"Validation loss: %.4f, validation accuracy: %.2f%%\" % (loss_val, acc_val))"
+    "#create dataloader for test data\n",
+    "# The dataloader makes our dataset iterable \n",
+    "batch_size_test = 256\n",
+    "test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=batch_size_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "acc_test, loss_test = evaluate(test_dataloader, linear_model, loss_fn)\n",
+    "print(\"Test loss: %.4f, test accuracy: %.2f%%\" % (loss_test, acc_test))"
    ]
   },
   {
@@ -396,7 +429,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Here are the first 10 images from the validation data that this small model classified to a wrong class:\n",
+    "Here are the first 10 images from the test data that this small model classified to a wrong class:\n",
     "\n"
    ]
   },
@@ -406,7 +439,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "show_failures(linear_model, val_dataloader)"
+    "show_failures(linear_model, test_dataloader)"
    ]
   },
   {
@@ -475,10 +508,10 @@
     "        self.layers_stack = nn.Sequential(\n",
     "            nn.Linear(28*28, 50),\n",
     "            nn.ReLU(),\n",
-    "            #nn.Dropout(0.2),\n",
+    "            nn.Dropout(0.2),\n",
     "            nn.Linear(50, 50),\n",
     "            nn.ReLU(),\n",
-    "            #nn.Dropout(0.2),\n",
+    "            nn.Dropout(0.2),\n",
     "            nn.Linear(50, 10)\n",
     "        )\n",
     "        \n",
@@ -489,23 +522,69 @@
     "        return x"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nonlinear_model = NonlinearClassifier()\n",
+    "loss_fn = nn.CrossEntropyLoss()\n",
+    "optimizer = torch.optim.SGD(nonlinear_model.parameters(), lr=0.05)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "\n",
+    "epochs = 5\n",
+    "train_loss_all = []\n",
+    "val_loss_all = []\n",
+    "for j in range(epochs):\n",
+    "    train_one_epoch(train_dataloader, nonlinear_model, loss_fn, optimizer)\n",
+    "    \n",
+    "    # checking on the training loss and accuracy once per epoch\n",
+    "    acc, loss = evaluate(train_dataloader, nonlinear_model, loss_fn)\n",
+    "    train_loss_all.append(loss)\n",
+    "    print(f\"Epoch {j}: training loss: {loss}, accuracy: {acc}\")\n",
+    "    \n",
+    "    # checking on the validation loss and accuracy once per epoch\n",
+    "    val_acc, val_loss = evaluate(val_dataloader, nonlinear_model, loss_fn)\n",
+    "    val_loss_all.append(val_loss)\n",
+    "    print(f\"Epoch {j}: val. loss: {val_loss}, val. accuracy: {val_acc}\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Homework: train a Nonlinear Classifier"
+    "# Homework"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "1. Write some code to train the NonlinearClassifier.\n",
-    "2. Create a data loader for the test data and check your model's accuracy on the test data. \n",
+    "1. Write code to train the NonlinearClassifier first with regularization on the weights (check the weight decay parameter https://pytorch.org/docs/stable/generated/torch.optim.SGD.html).\n",
+    "2. Compare the quality of your model when using different:\n",
+    " - batch sizes, \n",
+    " - learning rate,\n",
+    " - activation function. \n",
     "\n",
     "If you have time, experiment with how to improve the model. Note: training and validation data can be used to compare models, but test data should be saved until the end as a final check of generalization. "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -516,9 +595,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "common_pkgs",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "common_pkgs"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -530,7 +609,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.7.4"
   }
  },
  "nbformat": 4,