2b_batch_normalization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mCt-frpFH8oN"
   },
   "source": [
    "The tutorials use PyTorch. You will need to load the following dependencies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This specific version of torchvision is needed to download the mnist set\n",
    "!pip3 install torch torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hqnl0AKVXIA4"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import imageio\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import PIL\n",
    "import skimage.transform\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "from IPython import display\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial 2b: Batch Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CKeYuM-cIXxs"
   },
   "source": [
    "The code below may be helpful in visualizing PyTorch tensors as images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZZd_rI8edYIB"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "\n",
    "def show(img):\n",
    "    \"\"\"Show PyTorch tensor img as an image in matplotlib.\"\"\"\n",
    "    npimg = img.cpu().detach().numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)), interpolation=\"nearest\")\n",
    "    plt.grid(False)\n",
    "    plt.gca().axis(\"off\")\n",
    "\n",
    "\n",
    "def display_thumb(img):\n",
    "    display.display(transforms.Resize(128)(img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dzfEE578uSNp"
   },
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zgE0byUgKwM6"
   },
   "source": [
    "Load MNIST and define train/test functions as before. Please make sure you read the code carefully and understand what it is doing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 332,
     "referenced_widgets": [
      "a42490f444564fd7ac946d852614792f",
      "3373c12efcad42459f1d367d75ea3d46",
      "07dc9f747d104c11a43f01a95ed57902",
      "6e22ffc1689840a080a7a0306b6e40aa",
      "5ba9ed21d81a4642a60bf8a25043b231",
      "da77a43ce62847e095c45f116b46d575",
      "3b9341e0dd1942a590a8a75b47c4c02a",
      "af5aba6f88f94c03b40f93feaf5513b4",
      "efa04fca6187462393be792a87847ead",
      "34c560af3a2c4560af7d0ff51e3ba192",
      "927c1878303a4601a439af11796c860d",
      "ba3f6e39cfab49c2bdbf95eb8dcea867",
      "c6d63cfbab0d43ef97fd112250844b4b",
      "71de708bde354564bb46cdbd2254557c",
      "d83b581aeb674ca293e3b67ecdc9059f",
      "5cc2b2260d394c79abdac992228b0e31",
      "ef04916378e54e12806b08b0bcfbb8fd",
      "76bca89552e94a14a770b01d2ed5a632",
      "63cefcf9944d48cd8896bff26f85d8d0",
      "1b7ef3e5f8014502929183c185bde554",
      "053499a12c9a4c4ab3c2effba82aba70",
      "4dc4b3a5bc7d4d1896a459836fc7610f",
      "e95daa37066d4e369e3541e1816e07c1",
      "0542a250bd1a443c89cba5d525ee5e7b",
      "694a02b8ec6340b29c7bd53fd4a91e29",
      "8047845f5cf24c9889c98d227bdbab79",
      "8a64b78684d54b6bbc4949ac797d2ebe",
      "2e4528b2db2a497c89907cd6366ad893",
      "3bd8a63651ee4c538747d328c7b1e192",
      "f723b21b2c314b599f65d06b686693bf",
      "9954acb0661f4e438c7baf7e4a43f493",
      "f768eb9193774ea28185768e913147b9"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10900,
     "status": "ok",
     "timestamp": 1586534977022,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "-NMUce6PKu-o",
    "outputId": "de31434a-01c4-42c3-cdbe-df59992d0355"
   },
   "outputs": [],
   "source": [
    "# Load the training and test dataset.\n",
    "mnist_train = datasets.MNIST(\n",
    "    \"/tmp/mnist\", train=True, download=True, transform=transforms.ToTensor()\n",
    ")\n",
    "mnist_test = datasets.MNIST(\n",
    "    \"/tmp/mnist\", train=False, download=True, transform=transforms.ToTensor()\n",
    ")\n",
    "\n",
    "# Size of the batches the data loader will produce.\n",
    "batch_size = 64\n",
    "\n",
    "# This creates the dataloaders.\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    mnist_train, batch_size=batch_size, shuffle=True\n",
    ")\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    mnist_test, batch_size=batch_size, shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10899,
     "status": "ok",
     "timestamp": 1586534977023,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "TgAJ94UgK1VU",
    "outputId": "7aad0856-eaf6-4cb0-a70f-da77c43c4e7e"
   },
   "outputs": [],
   "source": [
    "def train(model, criterion, data_loader, optimizer, num_epochs):\n",
    "    \"\"\"Simple training loop for a PyTorch model.\"\"\"\n",
    "\n",
    "    # Make sure model is in training mode.\n",
    "    model.train()\n",
    "\n",
    "    # Move model to the device (CPU or GPU).\n",
    "    model.to(device)\n",
    "\n",
    "    # Exponential moving average of the loss.\n",
    "    ema_loss = None\n",
    "\n",
    "    # Loop over epochs.\n",
    "    for epoch in range(num_epochs):\n",
    "\n",
    "        # Loop over data.\n",
    "        for batch_idx, (data, target) in enumerate(data_loader):\n",
    "\n",
    "            # Forward pass.\n",
    "            output = model(data.to(device))\n",
    "            loss = criterion(output.to(device), target.to(device))\n",
    "\n",
    "            # Backward pass.\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # NOTE: It is important to call .item() on the loss before summing.\n",
    "            if ema_loss is None:\n",
    "                ema_loss = loss.item()\n",
    "            else:\n",
    "                ema_loss += (loss.item() - ema_loss) * 0.01\n",
    "\n",
    "        # Print out progress the end of epoch.\n",
    "        print(\n",
    "            \"Train Epoch: {} \\tLoss: {:.6f}\".format(epoch, ema_loss),\n",
    "        )\n",
    "\n",
    "\n",
    "def test(model, data_loader):\n",
    "    \"\"\"Measures the accuracy of a model on a data set.\"\"\"\n",
    "    # Make sure the model is in evaluation mode.\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "\n",
    "    # We do not need to maintain intermediate activations while testing.\n",
    "    with torch.no_grad():\n",
    "\n",
    "        # Loop over test data.\n",
    "        for data, target in data_loader:\n",
    "\n",
    "            # Forward pass.\n",
    "            output = model(data.to(device))\n",
    "\n",
    "            # Get the label corresponding to the highest predicted probability.\n",
    "            pred = output.argmax(dim=1, keepdim=True)\n",
    "\n",
    "            # Count number of correct predictions.\n",
    "            correct += pred.cpu().eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    # Print test accuracy.\n",
    "    percent = 100.0 * correct / len(data_loader.dataset)\n",
    "    print(f\"Accuracy: {correct} / {len(data_loader.dataset)} ({percent:.0f}%)\")\n",
    "    return percent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the last tutorial, you implemented a naive convolution. In this section you will implement your own version of forward pass of nn.Conv2d without using any of PyTorch's (or numpy's) pre-defined convolutional functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_forward_naive(x, w, b, conv_param):\n",
    "    \"\"\"\n",
    "    A naive Python implementation of a convolutional layer.\n",
    "    The input consists of N data points, each with C channels, height H and\n",
    "    width W. We convolve each input with F different filters, where each filter\n",
    "    spans all C channels and has height HH and width WW.\n",
    "    Input:\n",
    "    - x: Input data of shape (N, C, H, W)\n",
    "    - w: Filter weights of shape (F, C, HH, WW)\n",
    "    - b: Biases, of shape (F,)\n",
    "    - conv_param: A dictionary with the following keys:\n",
    "      - 'stride': The number of pixels between adjacent receptive fields in the\n",
    "        horizontal and vertical directions.\n",
    "      - 'pad': The number of pixels that will be used to zero-pad the input.\n",
    "\n",
    "    During padding, 'pad' zeros should be placed symmetrically (i.e., equally on both sides)\n",
    "    along the height and width axes of the input. Be careful not to modfiy the original\n",
    "    input x directly.\n",
    "    Returns an array.\n",
    "    - out: Output data, of shape (N, F, H', W') where H' and W' are given by\n",
    "      H' = 1 + (H + 2 * pad - HH) / stride\n",
    "      W' = 1 + (W + 2 * pad - WW) / stride\n",
    "    \"\"\"\n",
    "    out = None\n",
    "\n",
    "    N, C, H, W = x.shape\n",
    "    num_filters, _, filter_height, filter_width = w.shape\n",
    "    stride, pad = conv_param[\"stride\"], conv_param[\"pad\"]\n",
    "\n",
    "    # Check dimensions.\n",
    "    assert (W + 2 * pad - filter_width) % stride == 0, \"width does not work\"\n",
    "    assert (H + 2 * pad - filter_height) % stride == 0, \"height does not work\"\n",
    "\n",
    "    ###########################################################################\n",
    "    # TODO: Implement the forward pass of a convolutional layer without using #\n",
    "    #       nn.Conv2D or other implementations of convolutions. Instead, use  #\n",
    "    #       standard for- and while-loops to iterate over the tensors.        #\n",
    "    #                                                                         #\n",
    "    # Hint: you can use the function torch.nn.functional.pad for padding.     #\n",
    "    ###########################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can test your implementation by running the following testing code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make convolution module.\n",
    "w_shape = (3, 3, 4, 4)\n",
    "w = torch.linspace(-0.2, 0.3, steps=torch.prod(torch.tensor(w_shape))).reshape(w_shape)\n",
    "b = torch.linspace(-0.1, 0.2, steps=3)\n",
    "\n",
    "# Compute output of module and compare against reference values.\n",
    "x_shape = (2, 3, 4, 4)\n",
    "x = torch.linspace(-0.1, 0.5, steps=torch.prod(torch.tensor(x_shape))).reshape(x_shape)\n",
    "out = conv_forward_naive(x, w, b, {\"stride\": 2, \"pad\": 1})\n",
    "\n",
    "correct_out = torch.tensor(\n",
    "    [\n",
    "        [\n",
    "            [[-0.08759809, -0.10987781], [-0.18387192, -0.2109216]],\n",
    "            [[0.21027089, 0.21661097], [0.22847626, 0.23004637]],\n",
    "            [[0.50813986, 0.54309974], [0.64082444, 0.67101435]],\n",
    "        ],\n",
    "        [\n",
    "            [[-0.98053589, -1.03143541], [-1.19128892, -1.24695841]],\n",
    "            [[0.69108355, 0.66880383], [0.59480972, 0.56776003]],\n",
    "            [[2.36270298, 2.36904306], [2.38090835, 2.38247847]],\n",
    "        ],\n",
    "    ]\n",
    ")\n",
    "\n",
    "# Compare your output to ours; difference should be around e-8\n",
    "print(\"Testing conv_forward_naive\")\n",
    "rel_error = ((out - correct_out) / (out + correct_out + 1e-6)).mean()\n",
    "print(\"difference: \", rel_error)\n",
    "if abs(rel_error) < 1e-6:\n",
    "    print(\"Nice work! Your implementation of a convolution layer works correctly.\")\n",
    "else:\n",
    "    print(\n",
    "        \"Something is wrong. The output was expected to be {} but it was {}\".format(\n",
    "            correct_out, out\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GjTq5-k_q8s_"
   },
   "source": [
    "\n",
    "We will now replace the logistic regressor from the last tutorial by a small convolutional network with two convolutional layers and a linear layer, and ReLU activations in between the layers. Implement the model and use the same functions as before to train and test the convolutional network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11424,
     "status": "ok",
     "timestamp": 1586534977557,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "5t4hWoUYpp05",
    "outputId": "958f173f-ff66-466e-93d2-d8961a88b8a7"
   },
   "outputs": [],
   "source": [
    "class ConvolutionalNetwork(nn.Module):\n",
    "    \"\"\"Simple convolutional network.\"\"\"\n",
    "\n",
    "    def __init__(self, image_side_size, num_classes, in_channels=1):\n",
    "        super(ConvolutionalNetwork, self).__init__()\n",
    "\n",
    "        # Fill these in:\n",
    "        ##########################################################################\n",
    "        # TODO: Implement a convulutional and a linear part.                     #\n",
    "        # Hint: see forward() to understand how they should work together.       #\n",
    "        ##########################################################################\n",
    "        self.conv_network = None\n",
    "        self.linear = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_network(x)\n",
    "        x = self.linear(x.view(x.size(0), -1))\n",
    "        return x\n",
    "\n",
    "\n",
    "# Create and train convolutional network.\n",
    "# The accuracy should be around 96%.\n",
    "conv_model = ConvolutionalNetwork(28, 10)\n",
    "###########################################################################\n",
    "# TODO: Create criterion and optimize here.                               #\n",
    "###########################################################################\n",
    "criterion = None\n",
    "optimizer = None\n",
    "\n",
    "train(conv_model, criterion, train_loader, optimizer, num_epochs=5)\n",
    "test(conv_model, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspect the filters in the first layer of the trained convolutional network. What do they look like? Why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11418,
     "status": "ok",
     "timestamp": 1586534977560,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "LuSuQHT6bNdU",
    "outputId": "39179847-8353-4a8e-b5e3-e372c1bafcc7"
   },
   "outputs": [],
   "source": [
    "first_conv = list(conv_model.conv_network.children())[0]\n",
    "show(\n",
    "    torchvision.utils.make_grid(\n",
    "        first_conv.weight,\n",
    "        normalize=True,\n",
    "        nrow=8,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "71LRkRxndajG"
   },
   "source": [
    "## Batch Normalization\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oN8qTuGkZ-3X"
   },
   "source": [
    "Batch normalization is tenchique that allows to make training more stable fast [1].\n",
    "\n",
    "Below we define a convolutional network with 3 layers. Before each ReLU layer we insert a BatchNorm2d layer if `use_batch_norm` is `True`. This improves the convergence as guarantees as values have the same variance asn zero-means. As a result on average exactly half of the values will be nulled by ReLU.\n",
    "\n",
    "[1] Ioffe, Sergey, and Christian Szegedy. \"Batch normalization: Accelerating deep network training by reducing internal covariate shift.\" arXiv preprint arXiv:1502.03167 (2015).\n",
    "\n",
    "**Task**. Go ahead and add batch normalization layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vg4CO_WDeLqh"
   },
   "outputs": [],
   "source": [
    "class ConvolutionalNetworkWithBN(nn.Module):\n",
    "    \"\"\"Convolutional network uses batch normalization when `use_batch_norm` is `True`.\"\"\"\n",
    "\n",
    "    def __init__(self, use_batch_norm=False):\n",
    "        super().__init__()\n",
    "        num_channels = 8\n",
    "        self.use_batch_norm = use_batch_norm\n",
    "\n",
    "        # We define all our layers in a single Sequential. If use_batch_norm is\n",
    "        # True, we'd like maybe_batch_norm_layer to produce a batchnorm layer.\n",
    "        self.conv = nn.Sequential(\n",
    "            # 1x28x28 -> 8x24x24.\n",
    "            nn.Conv2d(1, num_channels, kernel_size=5),\n",
    "            # 8x24x24 -> 8x12x12.\n",
    "            nn.MaxPool2d(2),\n",
    "            self.maybe_batch_norm_layer(num_channels),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(num_channels, num_channels, kernel_size=3, padding=1),\n",
    "            self.maybe_batch_norm_layer(num_channels),\n",
    "            nn.ReLU(inplace=True),\n",
    "            # 8x12x12 -> 8x8x8.\n",
    "            nn.Conv2d(num_channels, num_channels, kernel_size=5),\n",
    "            # 8x8x8 -> 8x4x4.\n",
    "            nn.MaxPool2d(2),\n",
    "            self.maybe_batch_norm_layer(num_channels),\n",
    "            nn.ReLU(inplace=True),\n",
    "        )\n",
    "        self.linear = nn.Linear(num_channels * 4**2, 10)\n",
    "\n",
    "    def maybe_batch_norm_layer(self, num_channels):\n",
    "        if self.use_batch_norm:\n",
    "            ###########################################################################\n",
    "            # TODO: Add batch norm layer                                              #\n",
    "            # Put your code between START_GRADING and END_GRADING.                    #\n",
    "            ###########################################################################\n",
    "            pass\n",
    "        else:\n",
    "            # This layer will copy its input to the output.\n",
    "            return nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.linear(x.view(x.size(0), -1))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qbtzzBhiJ4PW"
   },
   "source": [
    "To see how batch normalization improves stability, let's try to train the neural network with different learning rates and check the accuracies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 58814,
     "status": "ok",
     "timestamp": 1586535024967,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "YXuSliJF3r7u",
    "outputId": "f53421dc-2979-4df1-f45b-37789bd79e69"
   },
   "outputs": [],
   "source": [
    "for lr in 0.3, 0.1, 0.03, 0.01, 0.003:\n",
    "    conv_model = ConvolutionalNetworkWithBN()\n",
    "    optimizer = torch.optim.SGD(conv_model.parameters(), lr=lr)\n",
    "    train(conv_model, nn.CrossEntropyLoss(), train_loader, optimizer, num_epochs=1)\n",
    "    accuracy = test(conv_model, test_loader)\n",
    "    if accuracy > 96:\n",
    "        print(f\"##### lr={lr} leads to GOOD accuracy (no batch normalization)\")\n",
    "    elif accuracy > 90:\n",
    "        print(f\"##### lr={lr} leads to DECENT accuracy (no batch normalization)\")\n",
    "    else:\n",
    "        print(f\"##### lr={lr} leads to BAD accuracy (no batch normalization)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WrJBxM03KOZ-"
   },
   "source": [
    "As you can see learning rate should be in a narrow region to get GOOD result.\n",
    "\n",
    "Now let's try to run the same batch normalization. Before running the code you should finish TODO in ConvolutionalNetworkWithBN code. If your code is correct, then accuracies will improve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 99368,
     "status": "ok",
     "timestamp": 1586535065529,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "6u2JJdnX5VKF",
    "outputId": "ad250aa4-e160-4b5f-cd04-a39b082750b0"
   },
   "outputs": [],
   "source": [
    "for lr in 0.3, 0.1, 0.03, 0.01, 0.003:\n",
    "    conv_model = ConvolutionalNetworkWithBN(use_batch_norm=True)\n",
    "    optimizer = torch.optim.SGD(conv_model.parameters(), lr=lr)\n",
    "    train(conv_model, nn.CrossEntropyLoss(), train_loader, optimizer, num_epochs=1)\n",
    "    accuracy = test(conv_model, test_loader)\n",
    "    if accuracy > 96:\n",
    "        print(f\"##### lr={lr} leads to GOOD accuracy (with batch normalization)\")\n",
    "    elif accuracy > 90:\n",
    "        print(f\"##### lr={lr} leads to DECENT accuracy (with batch normalization)\")\n",
    "    else:\n",
    "        print(f\"##### lr={lr} leads to BAD accuracy (with batch normalization)\")\n",
    "    if lr >= 0.01:\n",
    "        assert (\n",
    "            accuracy > 90\n",
    "        ), \"Accuracy is too low. Check that your BatchNorm implementation is correct!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gdgr7eNFTTAn"
   },
   "source": [
    "**Question [optional]:** Replace the batch normalization layer in your network by your own implementation. Confirm your batch-normalization implementation is correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 252
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 99376,
     "status": "error",
     "timestamp": 1586535065546,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "MAuhUMtjTj3f",
    "outputId": "66862e20-f71c-41bd-c591-7b0a65f35a60"
   },
   "outputs": [],
   "source": [
    "class MyBatchNorm2d(nn.Module):\n",
    "    \"\"\"Simple implementation of batch normalization.\"\"\"\n",
    "\n",
    "    def __init__(self, num_channels, momentum=0.1, epsilon=1e-5):\n",
    "        super(MyBatchNorm2d, self).__init__()\n",
    "\n",
    "        # Initialize bias and gain parameters.\n",
    "        self.gamma = nn.Parameter(torch.ones(1, num_channels, 1, 1))\n",
    "        self.beta = nn.Parameter(torch.zeros(1, num_channels, 1, 1))\n",
    "\n",
    "        # Initialize moving averages.\n",
    "        self.epsilon = epsilon\n",
    "        self.momentum = momentum\n",
    "        self.register_buffer(\"running_mean\", torch.zeros((1, num_channels, 1, 1)))\n",
    "        self.register_buffer(\"running_var\", torch.ones((1, num_channels, 1, 1)))\n",
    "\n",
    "    def forward(self, x):\n",
    "        # Check that input is of correct size.\n",
    "        assert x.dim() == 4, \"input should be NCHW\"\n",
    "        assert x.size(1) == self.gamma.numel()\n",
    "\n",
    "        ########################################################################\n",
    "        # TODO: Add batch norm layer implementation.                           #\n",
    "        # You code should:                                                     #\n",
    "        #   * Compute mean and var.                                            #\n",
    "        #   * Update running_mean and running_var.                             #\n",
    "        #   * Apply mean - variance normalization to x.                        #\n",
    "        # Put your code between START_GRADING and END_GRADING.                 #\n",
    "        ########################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use this code to test if your implementation is correct.\n",
    "batch_size, num_channels, im_size = 32, 8, 6\n",
    "batchnorm1 = nn.BatchNorm2d(num_channels)\n",
    "batchnorm2 = MyBatchNorm2d(num_channels)\n",
    "for key, param in batchnorm1.named_parameters():\n",
    "    if key == \"weight\":\n",
    "        param.data.fill_(1.0)  # undo random initialization in nn.BatchNorm2d\n",
    "for mode in [True, False]:  # test in training and evaluation mode\n",
    "    batchnorm1.train(mode=mode)\n",
    "    batchnorm2.train(mode=mode)\n",
    "    for _ in range(5):\n",
    "        x = torch.randn(batch_size, num_channels, im_size, im_size) + 10.0\n",
    "        out1 = batchnorm1(x)\n",
    "        out2 = batchnorm2(x)\n",
    "        assert (\n",
    "            batchnorm1.running_mean - batchnorm2.running_mean.squeeze()\n",
    "        ).abs().max() < 1e-5, \"running mean is incorrect (%s mode)\" % (\n",
    "            \"train\" if mode else \"eval\"\n",
    "        )\n",
    "        assert (\n",
    "            batchnorm1.running_var - batchnorm2.running_var.squeeze()\n",
    "        ).abs().max() < 1e-5, \"running variance is incorrect (%s mode)\" % (\n",
    "            \"train\" if mode else \"eval\"\n",
    "        )\n",
    "        assert (\n",
    "            out1 - out2\n",
    "        ).abs().max() < 5e-3, \"normalized output is incorrect (%s mode)\" % (\n",
    "            \"train\" if mode else \"eval\"\n",
    "        )\n",
    "print(\"All OK!\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "convnet_tutorial2_filled.ipynb",
   "provenance": [
    {
     "file_id": "1Eq_v8eTk4XAxFXhjL6rk26gwfwvpZYBn",
     "timestamp": 1553633151446
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "053499a12c9a4c4ab3c2effba82aba70": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "0542a250bd1a443c89cba5d525ee5e7b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "07dc9f747d104c11a43f01a95ed57902": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_da77a43ce62847e095c45f116b46d575",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5ba9ed21d81a4642a60bf8a25043b231",
      "value": 1
     }
    },
    "1b7ef3e5f8014502929183c185bde554": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0542a250bd1a443c89cba5d525ee5e7b",
      "placeholder": "​",
      "style": "IPY_MODEL_e95daa37066d4e369e3541e1816e07c1",
      "value": " 1654784/? [00:16&lt;00:00, 1174712.52it/s]"
     }
    },
    "2e4528b2db2a497c89907cd6366ad893": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f768eb9193774ea28185768e913147b9",
      "placeholder": "​",
      "style": "IPY_MODEL_9954acb0661f4e438c7baf7e4a43f493",
      "value": " 8192/? [00:00&lt;00:00, 11911.67it/s]"
     }
    },
    "3373c12efcad42459f1d367d75ea3d46": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34c560af3a2c4560af7d0ff51e3ba192": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3815582c26af4ee4a6031a103a196eff": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b9341e0dd1942a590a8a75b47c4c02a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3bd8a63651ee4c538747d328c7b1e192": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "45200009cb0f40449c05c889854d653c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4dc4b3a5bc7d4d1896a459836fc7610f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5ba9ed21d81a4642a60bf8a25043b231": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5cc2b2260d394c79abdac992228b0e31": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "63cefcf9944d48cd8896bff26f85d8d0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4dc4b3a5bc7d4d1896a459836fc7610f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_053499a12c9a4c4ab3c2effba82aba70",
      "value": 1
     }
    },
    "694a02b8ec6340b29c7bd53fd4a91e29": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8a64b78684d54b6bbc4949ac797d2ebe",
       "IPY_MODEL_2e4528b2db2a497c89907cd6366ad893"
      ],
      "layout": "IPY_MODEL_8047845f5cf24c9889c98d227bdbab79"
     }
    },
    "6e22ffc1689840a080a7a0306b6e40aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_af5aba6f88f94c03b40f93feaf5513b4",
      "placeholder": "​",
      "style": "IPY_MODEL_3b9341e0dd1942a590a8a75b47c4c02a",
      "value": " 9920512/? [00:20&lt;00:00, 4620495.53it/s]"
     }
    },
    "71de708bde354564bb46cdbd2254557c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7421dc7a0a874ab8835cdb30a73372ee": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a8773930a517488dbdbb30ac3bef7971",
      "placeholder": "​",
      "style": "IPY_MODEL_93f3ec87a84542ada9518c7771066cf3",
      "value": " 44.7M/44.7M [00:00&lt;00:00, 149MB/s]"
     }
    },
    "76bca89552e94a14a770b01d2ed5a632": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8047845f5cf24c9889c98d227bdbab79": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8a64b78684d54b6bbc4949ac797d2ebe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f723b21b2c314b599f65d06b686693bf",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3bd8a63651ee4c538747d328c7b1e192",
      "value": 1
     }
    },
    "927c1878303a4601a439af11796c860d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_71de708bde354564bb46cdbd2254557c",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c6d63cfbab0d43ef97fd112250844b4b",
      "value": 1
     }
    },
    "93f3ec87a84542ada9518c7771066cf3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9954acb0661f4e438c7baf7e4a43f493": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a18479993a7547d688eed72386289911": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a42490f444564fd7ac946d852614792f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_07dc9f747d104c11a43f01a95ed57902",
       "IPY_MODEL_6e22ffc1689840a080a7a0306b6e40aa"
      ],
      "layout": "IPY_MODEL_3373c12efcad42459f1d367d75ea3d46"
     }
    },
    "a8773930a517488dbdbb30ac3bef7971": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "af5aba6f88f94c03b40f93feaf5513b4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ba3f6e39cfab49c2bdbf95eb8dcea867": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5cc2b2260d394c79abdac992228b0e31",
      "placeholder": "​",
      "style": "IPY_MODEL_d83b581aeb674ca293e3b67ecdc9059f",
      "value": " 32768/? [00:00&lt;00:00, 52827.20it/s]"
     }
    },
    "bdc046fad38343b3aba8fd0fbaffb9c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3815582c26af4ee4a6031a103a196eff",
      "max": 46827520,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a18479993a7547d688eed72386289911",
      "value": 46827520
     }
    },
    "c6d63cfbab0d43ef97fd112250844b4b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "cb7a435d8db0472eb3388f5d0d53043f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bdc046fad38343b3aba8fd0fbaffb9c9",
       "IPY_MODEL_7421dc7a0a874ab8835cdb30a73372ee"
      ],
      "layout": "IPY_MODEL_45200009cb0f40449c05c889854d653c"
     }
    },
    "d83b581aeb674ca293e3b67ecdc9059f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "da77a43ce62847e095c45f116b46d575": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e95daa37066d4e369e3541e1816e07c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ef04916378e54e12806b08b0bcfbb8fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_63cefcf9944d48cd8896bff26f85d8d0",
       "IPY_MODEL_1b7ef3e5f8014502929183c185bde554"
      ],
      "layout": "IPY_MODEL_76bca89552e94a14a770b01d2ed5a632"
     }
    },
    "efa04fca6187462393be792a87847ead": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_927c1878303a4601a439af11796c860d",
       "IPY_MODEL_ba3f6e39cfab49c2bdbf95eb8dcea867"
      ],
      "layout": "IPY_MODEL_34c560af3a2c4560af7d0ff51e3ba192"
     }
    },
    "f723b21b2c314b599f65d06b686693bf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f768eb9193774ea28185768e913147b9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}