1a_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hqnl0AKVXIA4"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot\n",
    "import numpy.random\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "from torch import Tensor\n",
    "from torch.nn import Module\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import MNIST\n",
    "from torchvision.transforms import ToTensor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mCt-frpFH8oN"
   },
   "source": [
    "# Tutorial 1a. Logistic Regression\n",
    "\n",
    "In the first tutorial, we are going to train a logistic regressor on the MNIST dataset of handwritten digits. Next, we will turn this logistic regressor into a non-linear convolutional network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dzfEE578uSNp"
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    DEVICE = \"cuda:0\"\n",
    "else:\n",
    "    DEVICE = \"cpu\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Datasets\n",
    "\n",
    "The following code will load the MNIST dataset. Run it and inspect some of the images and their labels to confirm they are correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 349,
     "referenced_widgets": [
      "59a8e0ee48e14c9691224d013d6c46fe",
      "ee8b4317d860412fa60bca6d1d0c935a",
      "bcc108058dc4490986ec987c568b2034",
      "ab7e6d830cc74c14876a052480eb185a",
      "3a488dd9262d42ac9b119cd37e2e71c9",
      "229d85ab3245444c9ad5b57c6ec58688",
      "a9ccabebd2c6428594120631518c505c",
      "a739b97fb09848dbae2fb6d0fef09ef4",
      "41f12163cbd84febb7fade8126ed7d5e",
      "48dc62d211994fb1b3b323da3c8c8a26",
      "8f7373518a7840b38d00de7608a9e218",
      "e92daa0c7043415b8463dffd654e93e5",
      "df0848e4bca84d95b178cbe955417bc4",
      "10c91bce2e8b42c587557b0a649a79fe",
      "1a38c10f1d0e4f20a4958d453757f9e7",
      "70658583c62a40c28345d43ceb70810b",
      "e488365503444d9c8313fd2017d05f9e",
      "d677eff7d5a4494087a60ff1db4dab92",
      "200b60fab8514008a571603c6a2519f8",
      "34a38f1355e54a24ad0757e91e016ff4",
      "8f98aca9cfae490dab5dccf54b939ce0",
      "04f7f68d4f784cf19f942ee29c24ecc7",
      "99bb01a7fb144b149460669c65b22654",
      "466dc3fe5f2c447ebc9f7259e886e606",
      "e11d5108f5a74dd6b7143172c693dc6a",
      "590b36901152442088a4c5e05e781b3d",
      "2de5166b2a40479fb69c08e7fda49594",
      "655321e930014d03a986ab862d9c01a1",
      "bcbba5e1db1d498eb4b9ce545ff26d8a",
      "d57756abec8742db9d650c814b474137",
      "3c129350f337408f920706b3836528b9",
      "78f9f8dd4275405b96194f2507dae175"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7621,
     "status": "ok",
     "timestamp": 1586535092902,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "GX_ky5qC--uv",
    "outputId": "97a811cf-59f5-4bb5-be96-03e19190cd4b"
   },
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(\n",
    "    dataset=MNIST(\n",
    "        root=\"/tmp/mnist\",\n",
    "        train=True,\n",
    "        transform=ToTensor(),\n",
    "        download=True,\n",
    "    ),\n",
    "    batch_size=64,\n",
    "    shuffle=True,\n",
    ")\n",
    "\n",
    "test_dataloader = DataLoader(\n",
    "    dataset=MNIST(\n",
    "        root=\"/tmp/mnist\",\n",
    "        train=False,\n",
    "        transform=ToTensor(),\n",
    "        download=True,\n",
    "    ),\n",
    "    batch_size=64,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 62
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7559,
     "status": "ok",
     "timestamp": 1586535092902,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "E8dy7-kBHwco",
    "outputId": "66a535b5-83bc-48e0-f94a-1a79de8c3e66"
   },
   "outputs": [],
   "source": [
    "image, target = [*test_dataloader][0]\n",
    "\n",
    "random_index = numpy.random.randint(0, 64)\n",
    "\n",
    "image, target = image[random_index, 0], target[random_index]\n",
    "\n",
    "matplotlib.pyplot.imshow(\n",
    "    image, \n",
    "    cmap=\"gray\", \n",
    "    interpolation=\"nearest\",\n",
    ")\n",
    "\n",
    "matplotlib.pyplot.title(f\"target = {target}\")\n",
    "matplotlib.pyplot.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aKbXgidhWr6L"
   },
   "source": [
    "Next, we create a PyTorch dataloader for the MNIST dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "liekFZzvYX9E"
   },
   "source": [
    "Next, implement a logistic regression model in PyTorch. Note that a logistic regressor uses a linear transformation of the input.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 1. Logistic Regression Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zEw5YorSYkWF"
   },
   "outputs": [],
   "source": [
    "# class LogisticRegression(Module):\n",
    "#     def __init__(self, input_size: int, num_classes: int):\n",
    "#         super().__init__()\n",
    "#y = ... \n",    
    "#         ###########################################################################\n",
    "#         # TODO: Instantiate the layer here.                                       #\n",
    "#         ###########################################################################\n",
    "# \n",
    "#     def forward(self, x: Tensor) -> Tensor:\n",
    "#         ###########################################################################\n",
    "#         # TODO: Apply the layer to the input.                                     #\n",
    "#         ###########################################################################\n",
    "#         pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "module = LogisticRegression(28 * 28, 10)\n",
    "\n",
    "module = module.to(device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###########################################################################\n",
    "# TODO: Create criterion and optimizer here.                              #\n",
    "###########################################################################\n",
    "\n",
    "criterion = \n",
    "\n",
    "optimizer = "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use the following generic training loop for a PyTorch model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qVyEKl3OZLJw"
   },
   "outputs": [],
   "source": [
    "# Train the model. If everything is correct, the loss should go below 0.45.\n",
    "\n",
    "EPOCHS = 5\n",
    "\n",
    "# Exponential moving average of the loss:\n",
    "ema = None\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    for batch_index, (train_images, train_targets) in enumerate(train_dataloader):\n",
    "        train_images = train_images.view(-1, 28 * 28).requires_grad_().to(device=DEVICE)\n",
    "\n",
    "        train_targets = train_targets.to(device=DEVICE)\n",
    "\n",
    "        # Clear gradients w.r.t. parameters\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Forward pass to get output/logits\n",
    "        outputs = module(train_images)\n",
    "\n",
    "        # Calculate Loss: softmax --> cross entropy loss\n",
    "        loss = criterion(outputs, train_targets)\n",
    "\n",
    "        # Getting gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "\n",
    "        # Updates parameters:\n",
    "        optimizer.step()\n",
    "\n",
    "        # NOTE: It is important to call .item() on the loss before summing.\n",
    "        if ema is None:\n",
    "            ema = loss.item()\n",
    "        else:\n",
    "            ema += (loss.item() - ema) * 0.01\n",
    "\n",
    "        if batch_index % 500 == 0:\n",
    "            print(\n",
    "                \"Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}\".format(\n",
    "                    epoch,\n",
    "                    batch_index * len(train_images),\n",
    "                    len(train_dataloader.dataset),\n",
    "                    100.0 * batch_index / len(train_dataloader),\n",
    "                    ema,\n",
    "                ),\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oN8qTuGkZ-3X"
   },
   "source": [
    "**Question:** For the model you are currently using, is there any difference between using the model in `train` mode or using it in `eval` mode? \n",
    "\n",
    "Create an SGD optimizer and us it to train the logistic regressor on the MNIST training data for a few epochs. What loss function do you need to use?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x0QN7Mhiar7d"
   },
   "source": [
    "### Embeddings \n",
    "\n",
    "Visualize the weights of the trained model. What do you see? Why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert module.y.weight.shape == (10, 28 * 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 167
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 37796,
     "status": "ok",
     "timestamp": 1586535123165,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "gxletH44a4MX",
    "outputId": "1ba6c589-bcbb-4b86-a9ab-7fd4bc5bdfa1"
   },
   "outputs": [],
   "source": [
    "matplotlib.pyplot.imshow(\n",
    "    numpy.transpose(\n",
    "        torchvision.utils.make_grid(\n",
    "            module.y.weight.view.cpu()(10, 1, 28, 28),\n",
    "            normalize=True,\n",
    "            nrow=5,\n",
    "        ), \n",
    "        (1, 2, 0),\n",
    "    ), \n",
    "    interpolation=\"nearest\",\n",
    ")\n",
    "\n",
    "matplotlib.pyplot.grid(False)\n",
    "matplotlib.pyplot.gca().axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Lxf5NxQ6a5cT"
   },
   "source": [
    "Use the following function to measure the test accuracy of your trained model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8UYWbqZYa9Qr"
   },
   "outputs": [],
   "source": [
    "correct_predictions = 0\n",
    "predictions = 0\n",
    "\n",
    "# Iterate through test dataset\n",
    "for test_images, test_targets in test_dataloader:\n",
    "    test_images = test_images.view(-1, 28 * 28).to(DEVICE)\n",
    "\n",
    "    # Forward pass only to get logits/output\n",
    "    outputs = module(test_images)\n",
    "\n",
    "    # Get predictions from the maximum value\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "\n",
    "    predictions += test_targets.size(0)\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        correct_predictions += (predicted.cpu() == test_targets.cpu()).sum()\n",
    "    else:\n",
    "        correct_predictions += (predicted == test_targets).sum()\n",
    "\n",
    "correct_predictions.item() / predictions"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "convnet_tutorial1_filled.ipynb",
   "provenance": [
    {
     "file_id": "1Eq_v8eTk4XAxFXhjL6rk26gwfwvpZYBn",
     "timestamp": 1553454632964
    }
   ]
  },
  "kernelspec": {
   "display_name": "convnet_tutorials",
   "language": "python",
   "name": "convnet_tutorials"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "04f7f68d4f784cf19f942ee29c24ecc7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "10c91bce2e8b42c587557b0a649a79fe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1a38c10f1d0e4f20a4958d453757f9e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "200b60fab8514008a571603c6a2519f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_04f7f68d4f784cf19f942ee29c24ecc7",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8f98aca9cfae490dab5dccf54b939ce0",
      "value": 1
     }
    },
    "229d85ab3245444c9ad5b57c6ec58688": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2de5166b2a40479fb69c08e7fda49594": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "  0%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d57756abec8742db9d650c814b474137",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_bcbba5e1db1d498eb4b9ce545ff26d8a",
      "value": 0
     }
    },
    "34a38f1355e54a24ad0757e91e016ff4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_466dc3fe5f2c447ebc9f7259e886e606",
      "placeholder": "​",
      "style": "IPY_MODEL_99bb01a7fb144b149460669c65b22654",
      "value": " 1654784/? [00:00&lt;00:00, 1924711.94it/s]"
     }
    },
    "3a488dd9262d42ac9b119cd37e2e71c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3c129350f337408f920706b3836528b9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "41f12163cbd84febb7fade8126ed7d5e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8f7373518a7840b38d00de7608a9e218",
       "IPY_MODEL_e92daa0c7043415b8463dffd654e93e5"
      ],
      "layout": "IPY_MODEL_48dc62d211994fb1b3b323da3c8c8a26"
     }
    },
    "466dc3fe5f2c447ebc9f7259e886e606": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "48dc62d211994fb1b3b323da3c8c8a26": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "590b36901152442088a4c5e05e781b3d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "59a8e0ee48e14c9691224d013d6c46fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bcc108058dc4490986ec987c568b2034",
       "IPY_MODEL_ab7e6d830cc74c14876a052480eb185a"
      ],
      "layout": "IPY_MODEL_ee8b4317d860412fa60bca6d1d0c935a"
     }
    },
    "655321e930014d03a986ab862d9c01a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_78f9f8dd4275405b96194f2507dae175",
      "placeholder": "​",
      "style": "IPY_MODEL_3c129350f337408f920706b3836528b9",
      "value": " 0/4542 [00:00&lt;?, ?it/s]"
     }
    },
    "70658583c62a40c28345d43ceb70810b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "78f9f8dd4275405b96194f2507dae175": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8f7373518a7840b38d00de7608a9e218": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_10c91bce2e8b42c587557b0a649a79fe",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_df0848e4bca84d95b178cbe955417bc4",
      "value": 1
     }
    },
    "8f98aca9cfae490dab5dccf54b939ce0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "99bb01a7fb144b149460669c65b22654": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a739b97fb09848dbae2fb6d0fef09ef4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a9ccabebd2c6428594120631518c505c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ab7e6d830cc74c14876a052480eb185a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a739b97fb09848dbae2fb6d0fef09ef4",
      "placeholder": "​",
      "style": "IPY_MODEL_a9ccabebd2c6428594120631518c505c",
      "value": " 9920512/? [00:20&lt;00:00, 1061366.86it/s]"
     }
    },
    "bcbba5e1db1d498eb4b9ce545ff26d8a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "bcc108058dc4490986ec987c568b2034": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_229d85ab3245444c9ad5b57c6ec58688",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3a488dd9262d42ac9b119cd37e2e71c9",
      "value": 1
     }
    },
    "d57756abec8742db9d650c814b474137": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d677eff7d5a4494087a60ff1db4dab92": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "df0848e4bca84d95b178cbe955417bc4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e11d5108f5a74dd6b7143172c693dc6a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2de5166b2a40479fb69c08e7fda49594",
       "IPY_MODEL_655321e930014d03a986ab862d9c01a1"
      ],
      "layout": "IPY_MODEL_590b36901152442088a4c5e05e781b3d"
     }
    },
    "e488365503444d9c8313fd2017d05f9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_200b60fab8514008a571603c6a2519f8",
       "IPY_MODEL_34a38f1355e54a24ad0757e91e016ff4"
      ],
      "layout": "IPY_MODEL_d677eff7d5a4494087a60ff1db4dab92"
     }
    },
    "e92daa0c7043415b8463dffd654e93e5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_70658583c62a40c28345d43ceb70810b",
      "placeholder": "​",
      "style": "IPY_MODEL_1a38c10f1d0e4f20a4958d453757f9e7",
      "value": " 32768/? [00:01&lt;00:00, 28639.26it/s]"
     }
    },
    "ee8b4317d860412fa60bca6d1d0c935a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}